2 * Copyright (c) 2010-2011 Picochip Ltd., Jamie Iles
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * You should have received a copy of the GNU General Public License
15 * along with this program; if not, write to the Free Software
16 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18 #include <crypto/internal/aead.h>
19 #include <crypto/aes.h>
20 #include <crypto/algapi.h>
21 #include <crypto/authenc.h>
22 #include <crypto/des.h>
23 #include <crypto/md5.h>
24 #include <crypto/sha.h>
25 #include <crypto/internal/skcipher.h>
26 #include <linux/clk.h>
27 #include <linux/crypto.h>
28 #include <linux/delay.h>
29 #include <linux/dma-mapping.h>
30 #include <linux/dmapool.h>
31 #include <linux/err.h>
32 #include <linux/init.h>
33 #include <linux/interrupt.h>
35 #include <linux/list.h>
36 #include <linux/module.h>
38 #include <linux/platform_device.h>
40 #include <linux/rtnetlink.h>
41 #include <linux/scatterlist.h>
42 #include <linux/sched.h>
43 #include <linux/sizes.h>
44 #include <linux/slab.h>
45 #include <linux/timer.h>
47 #include "picoxcell_crypto_regs.h"
50 * The threshold for the number of entries in the CMD FIFO available before
51 * the CMD0_CNT interrupt is raised. Increasing this value will reduce the
52 * number of interrupts raised to the CPU.
54 #define CMD0_IRQ_THRESHOLD 1
57 * The timeout period (in jiffies) for a PDU. When the the number of PDUs in
58 * flight is greater than the STAT_IRQ_THRESHOLD or 0 the timer is disabled.
59 * When there are packets in flight but lower than the threshold, we enable
60 * the timer and at expiry, attempt to remove any processed packets from the
61 * queue and if there are still packets left, schedule the timer again.
63 #define PACKET_TIMEOUT 1
65 /* The priority to register each algorithm with. */
66 #define SPACC_CRYPTO_ALG_PRIORITY 10000
68 #define SPACC_CRYPTO_KASUMI_F8_KEY_LEN 16
69 #define SPACC_CRYPTO_IPSEC_CIPHER_PG_SZ 64
70 #define SPACC_CRYPTO_IPSEC_HASH_PG_SZ 64
71 #define SPACC_CRYPTO_IPSEC_MAX_CTXS 32
72 #define SPACC_CRYPTO_IPSEC_FIFO_SZ 32
73 #define SPACC_CRYPTO_L2_CIPHER_PG_SZ 64
74 #define SPACC_CRYPTO_L2_HASH_PG_SZ 64
75 #define SPACC_CRYPTO_L2_MAX_CTXS 128
76 #define SPACC_CRYPTO_L2_FIFO_SZ 128
78 #define MAX_DDT_LEN 16
80 /* DDT format. This must match the hardware DDT format exactly. */
87 * Asynchronous crypto request structure.
89 * This structure defines a request that is either queued for processing or
93 struct list_head list
;
94 struct spacc_engine
*engine
;
95 struct crypto_async_request
*req
;
99 dma_addr_t src_addr
, dst_addr
;
100 struct spacc_ddt
*src_ddt
, *dst_ddt
;
101 void (*complete
)(struct spacc_req
*req
);
105 unsigned long ctrl_default
;
108 struct spacc_engine
*engine
;
109 struct list_head entry
;
114 struct spacc_engine
{
116 struct list_head pending
;
120 struct list_head completed
;
121 struct list_head in_progress
;
122 struct tasklet_struct complete
;
123 unsigned long fifo_sz
;
124 void __iomem
*cipher_ctx_base
;
125 void __iomem
*hash_key_base
;
126 struct spacc_alg
*algs
;
128 struct list_head registered_algs
;
129 struct spacc_aead
*aeads
;
131 struct list_head registered_aeads
;
138 struct timer_list packet_timeout
;
139 unsigned stat_irq_thresh
;
140 struct dma_pool
*req_pool
;
143 /* Algorithm type mask. */
144 #define SPACC_CRYPTO_ALG_MASK 0x7
146 /* SPACC definition of a crypto algorithm. */
148 unsigned long ctrl_default
;
150 struct crypto_alg alg
;
151 struct spacc_engine
*engine
;
152 struct list_head entry
;
157 /* Generic context structure for any algorithm type. */
158 struct spacc_generic_ctx
{
159 struct spacc_engine
*engine
;
165 /* Block cipher context. */
166 struct spacc_ablk_ctx
{
167 struct spacc_generic_ctx generic
;
168 u8 key
[AES_MAX_KEY_SIZE
];
171 * The fallback cipher. If the operation can't be done in hardware,
172 * fallback to a software version.
174 struct crypto_ablkcipher
*sw_cipher
;
177 /* AEAD cipher context. */
178 struct spacc_aead_ctx
{
179 struct spacc_generic_ctx generic
;
180 u8 cipher_key
[AES_MAX_KEY_SIZE
];
181 u8 hash_ctx
[SPACC_CRYPTO_IPSEC_HASH_PG_SZ
];
184 struct crypto_aead
*sw_cipher
;
187 static int spacc_ablk_submit(struct spacc_req
*req
);
189 static inline struct spacc_alg
*to_spacc_alg(struct crypto_alg
*alg
)
191 return alg
? container_of(alg
, struct spacc_alg
, alg
) : NULL
;
194 static inline struct spacc_aead
*to_spacc_aead(struct aead_alg
*alg
)
196 return container_of(alg
, struct spacc_aead
, alg
);
199 static inline int spacc_fifo_cmd_full(struct spacc_engine
*engine
)
201 u32 fifo_stat
= readl(engine
->regs
+ SPA_FIFO_STAT_REG_OFFSET
);
203 return fifo_stat
& SPA_FIFO_CMD_FULL
;
207 * Given a cipher context, and a context number, get the base address of the
210 * Returns the address of the context page where the key/context may
213 static inline void __iomem
*spacc_ctx_page_addr(struct spacc_generic_ctx
*ctx
,
217 return is_cipher_ctx
? ctx
->engine
->cipher_ctx_base
+
218 (indx
* ctx
->engine
->cipher_pg_sz
) :
219 ctx
->engine
->hash_key_base
+ (indx
* ctx
->engine
->hash_pg_sz
);
222 /* The context pages can only be written with 32-bit accesses. */
223 static inline void memcpy_toio32(u32 __iomem
*dst
, const void *src
,
226 const u32
*src32
= (const u32
*) src
;
229 writel(*src32
++, dst
++);
232 static void spacc_cipher_write_ctx(struct spacc_generic_ctx
*ctx
,
233 void __iomem
*page_addr
, const u8
*key
,
234 size_t key_len
, const u8
*iv
, size_t iv_len
)
236 void __iomem
*key_ptr
= page_addr
+ ctx
->key_offs
;
237 void __iomem
*iv_ptr
= page_addr
+ ctx
->iv_offs
;
239 memcpy_toio32(key_ptr
, key
, key_len
/ 4);
240 memcpy_toio32(iv_ptr
, iv
, iv_len
/ 4);
244 * Load a context into the engines context memory.
246 * Returns the index of the context page where the context was loaded.
248 static unsigned spacc_load_ctx(struct spacc_generic_ctx
*ctx
,
249 const u8
*ciph_key
, size_t ciph_len
,
250 const u8
*iv
, size_t ivlen
, const u8
*hash_key
,
253 unsigned indx
= ctx
->engine
->next_ctx
++;
254 void __iomem
*ciph_page_addr
, *hash_page_addr
;
256 ciph_page_addr
= spacc_ctx_page_addr(ctx
, indx
, 1);
257 hash_page_addr
= spacc_ctx_page_addr(ctx
, indx
, 0);
259 ctx
->engine
->next_ctx
&= ctx
->engine
->fifo_sz
- 1;
260 spacc_cipher_write_ctx(ctx
, ciph_page_addr
, ciph_key
, ciph_len
, iv
,
262 writel(ciph_len
| (indx
<< SPA_KEY_SZ_CTX_INDEX_OFFSET
) |
263 (1 << SPA_KEY_SZ_CIPHER_OFFSET
),
264 ctx
->engine
->regs
+ SPA_KEY_SZ_REG_OFFSET
);
267 memcpy_toio32(hash_page_addr
, hash_key
, hash_len
/ 4);
268 writel(hash_len
| (indx
<< SPA_KEY_SZ_CTX_INDEX_OFFSET
),
269 ctx
->engine
->regs
+ SPA_KEY_SZ_REG_OFFSET
);
275 static inline void ddt_set(struct spacc_ddt
*ddt
, dma_addr_t phys
, size_t len
)
282 * Take a crypto request and scatterlists for the data and turn them into DDTs
283 * for passing to the crypto engines. This also DMA maps the data so that the
284 * crypto engines can DMA to/from them.
286 static struct spacc_ddt
*spacc_sg_to_ddt(struct spacc_engine
*engine
,
287 struct scatterlist
*payload
,
289 enum dma_data_direction dir
,
290 dma_addr_t
*ddt_phys
)
292 unsigned mapped_ents
;
293 struct scatterlist
*cur
;
294 struct spacc_ddt
*ddt
;
298 nents
= sg_nents_for_len(payload
, nbytes
);
300 dev_err(engine
->dev
, "Invalid numbers of SG.\n");
303 mapped_ents
= dma_map_sg(engine
->dev
, payload
, nents
, dir
);
305 if (mapped_ents
+ 1 > MAX_DDT_LEN
)
308 ddt
= dma_pool_alloc(engine
->req_pool
, GFP_ATOMIC
, ddt_phys
);
312 for_each_sg(payload
, cur
, mapped_ents
, i
)
313 ddt_set(&ddt
[i
], sg_dma_address(cur
), sg_dma_len(cur
));
314 ddt_set(&ddt
[mapped_ents
], 0, 0);
319 dma_unmap_sg(engine
->dev
, payload
, nents
, dir
);
323 static int spacc_aead_make_ddts(struct aead_request
*areq
)
325 struct crypto_aead
*aead
= crypto_aead_reqtfm(areq
);
326 struct spacc_req
*req
= aead_request_ctx(areq
);
327 struct spacc_engine
*engine
= req
->engine
;
328 struct spacc_ddt
*src_ddt
, *dst_ddt
;
330 int src_nents
, dst_nents
;
331 struct scatterlist
*cur
;
332 int i
, dst_ents
, src_ents
;
334 total
= areq
->assoclen
+ areq
->cryptlen
;
336 total
+= crypto_aead_authsize(aead
);
338 src_nents
= sg_nents_for_len(areq
->src
, total
);
340 dev_err(engine
->dev
, "Invalid numbers of src SG.\n");
343 if (src_nents
+ 1 > MAX_DDT_LEN
)
347 if (areq
->src
!= areq
->dst
) {
348 dst_nents
= sg_nents_for_len(areq
->dst
, total
);
350 dev_err(engine
->dev
, "Invalid numbers of dst SG.\n");
353 if (src_nents
+ 1 > MAX_DDT_LEN
)
357 src_ddt
= dma_pool_alloc(engine
->req_pool
, GFP_ATOMIC
, &req
->src_addr
);
361 dst_ddt
= dma_pool_alloc(engine
->req_pool
, GFP_ATOMIC
, &req
->dst_addr
);
365 req
->src_ddt
= src_ddt
;
366 req
->dst_ddt
= dst_ddt
;
369 src_ents
= dma_map_sg(engine
->dev
, areq
->src
, src_nents
,
374 dst_ents
= dma_map_sg(engine
->dev
, areq
->dst
, dst_nents
,
378 dma_unmap_sg(engine
->dev
, areq
->src
, src_nents
,
383 src_ents
= dma_map_sg(engine
->dev
, areq
->src
, src_nents
,
391 * Now map in the payload for the source and destination and terminate
392 * with the NULL pointers.
394 for_each_sg(areq
->src
, cur
, src_ents
, i
)
395 ddt_set(src_ddt
++, sg_dma_address(cur
), sg_dma_len(cur
));
397 /* For decryption we need to skip the associated data. */
398 total
= req
->is_encrypt
? 0 : areq
->assoclen
;
399 for_each_sg(areq
->dst
, cur
, dst_ents
, i
) {
400 unsigned len
= sg_dma_len(cur
);
407 ddt_set(dst_ddt
++, sg_dma_address(cur
) + total
, len
- total
);
410 ddt_set(src_ddt
, 0, 0);
411 ddt_set(dst_ddt
, 0, 0);
416 dma_pool_free(engine
->req_pool
, dst_ddt
, req
->dst_addr
);
418 dma_pool_free(engine
->req_pool
, src_ddt
, req
->src_addr
);
423 static void spacc_aead_free_ddts(struct spacc_req
*req
)
425 struct aead_request
*areq
= container_of(req
->req
, struct aead_request
,
427 struct crypto_aead
*aead
= crypto_aead_reqtfm(areq
);
428 unsigned total
= areq
->assoclen
+ areq
->cryptlen
+
429 (req
->is_encrypt
? crypto_aead_authsize(aead
) : 0);
430 struct spacc_aead_ctx
*aead_ctx
= crypto_aead_ctx(aead
);
431 struct spacc_engine
*engine
= aead_ctx
->generic
.engine
;
432 int nents
= sg_nents_for_len(areq
->src
, total
);
434 /* sg_nents_for_len should not fail since it works when mapping sg */
435 if (unlikely(nents
< 0)) {
436 dev_err(engine
->dev
, "Invalid numbers of src SG.\n");
440 if (areq
->src
!= areq
->dst
) {
441 dma_unmap_sg(engine
->dev
, areq
->src
, nents
, DMA_TO_DEVICE
);
442 nents
= sg_nents_for_len(areq
->dst
, total
);
443 if (unlikely(nents
< 0)) {
444 dev_err(engine
->dev
, "Invalid numbers of dst SG.\n");
447 dma_unmap_sg(engine
->dev
, areq
->dst
, nents
, DMA_FROM_DEVICE
);
449 dma_unmap_sg(engine
->dev
, areq
->src
, nents
, DMA_BIDIRECTIONAL
);
451 dma_pool_free(engine
->req_pool
, req
->src_ddt
, req
->src_addr
);
452 dma_pool_free(engine
->req_pool
, req
->dst_ddt
, req
->dst_addr
);
455 static void spacc_free_ddt(struct spacc_req
*req
, struct spacc_ddt
*ddt
,
456 dma_addr_t ddt_addr
, struct scatterlist
*payload
,
457 unsigned nbytes
, enum dma_data_direction dir
)
459 int nents
= sg_nents_for_len(payload
, nbytes
);
462 dev_err(req
->engine
->dev
, "Invalid numbers of SG.\n");
466 dma_unmap_sg(req
->engine
->dev
, payload
, nents
, dir
);
467 dma_pool_free(req
->engine
->req_pool
, ddt
, ddt_addr
);
470 static int spacc_aead_setkey(struct crypto_aead
*tfm
, const u8
*key
,
473 struct spacc_aead_ctx
*ctx
= crypto_aead_ctx(tfm
);
474 struct crypto_authenc_keys keys
;
477 crypto_aead_clear_flags(ctx
->sw_cipher
, CRYPTO_TFM_REQ_MASK
);
478 crypto_aead_set_flags(ctx
->sw_cipher
, crypto_aead_get_flags(tfm
) &
479 CRYPTO_TFM_REQ_MASK
);
480 err
= crypto_aead_setkey(ctx
->sw_cipher
, key
, keylen
);
481 crypto_aead_clear_flags(tfm
, CRYPTO_TFM_RES_MASK
);
482 crypto_aead_set_flags(tfm
, crypto_aead_get_flags(ctx
->sw_cipher
) &
483 CRYPTO_TFM_RES_MASK
);
487 if (crypto_authenc_extractkeys(&keys
, key
, keylen
) != 0)
490 if (keys
.enckeylen
> AES_MAX_KEY_SIZE
)
493 if (keys
.authkeylen
> sizeof(ctx
->hash_ctx
))
496 memcpy(ctx
->cipher_key
, keys
.enckey
, keys
.enckeylen
);
497 ctx
->cipher_key_len
= keys
.enckeylen
;
499 memcpy(ctx
->hash_ctx
, keys
.authkey
, keys
.authkeylen
);
500 ctx
->hash_key_len
= keys
.authkeylen
;
505 crypto_aead_set_flags(tfm
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
509 static int spacc_aead_setauthsize(struct crypto_aead
*tfm
,
510 unsigned int authsize
)
512 struct spacc_aead_ctx
*ctx
= crypto_tfm_ctx(crypto_aead_tfm(tfm
));
514 return crypto_aead_setauthsize(ctx
->sw_cipher
, authsize
);
518 * Check if an AEAD request requires a fallback operation. Some requests can't
519 * be completed in hardware because the hardware may not support certain key
520 * sizes. In these cases we need to complete the request in software.
522 static int spacc_aead_need_fallback(struct aead_request
*aead_req
)
524 struct crypto_aead
*aead
= crypto_aead_reqtfm(aead_req
);
525 struct aead_alg
*alg
= crypto_aead_alg(aead
);
526 struct spacc_aead
*spacc_alg
= to_spacc_aead(alg
);
527 struct spacc_aead_ctx
*ctx
= crypto_aead_ctx(aead
);
530 * If we have a non-supported key-length, then we need to do a
533 if ((spacc_alg
->ctrl_default
& SPACC_CRYPTO_ALG_MASK
) ==
534 SPA_CTRL_CIPH_ALG_AES
&&
535 ctx
->cipher_key_len
!= AES_KEYSIZE_128
&&
536 ctx
->cipher_key_len
!= AES_KEYSIZE_256
)
542 static int spacc_aead_do_fallback(struct aead_request
*req
, unsigned alg_type
,
545 struct crypto_tfm
*old_tfm
= crypto_aead_tfm(crypto_aead_reqtfm(req
));
546 struct spacc_aead_ctx
*ctx
= crypto_tfm_ctx(old_tfm
);
547 struct aead_request
*subreq
= aead_request_ctx(req
);
549 aead_request_set_tfm(subreq
, ctx
->sw_cipher
);
550 aead_request_set_callback(subreq
, req
->base
.flags
,
551 req
->base
.complete
, req
->base
.data
);
552 aead_request_set_crypt(subreq
, req
->src
, req
->dst
, req
->cryptlen
,
554 aead_request_set_ad(subreq
, req
->assoclen
);
556 return is_encrypt
? crypto_aead_encrypt(subreq
) :
557 crypto_aead_decrypt(subreq
);
560 static void spacc_aead_complete(struct spacc_req
*req
)
562 spacc_aead_free_ddts(req
);
563 req
->req
->complete(req
->req
, req
->result
);
566 static int spacc_aead_submit(struct spacc_req
*req
)
568 struct aead_request
*aead_req
=
569 container_of(req
->req
, struct aead_request
, base
);
570 struct crypto_aead
*aead
= crypto_aead_reqtfm(aead_req
);
571 unsigned int authsize
= crypto_aead_authsize(aead
);
572 struct spacc_aead_ctx
*ctx
= crypto_aead_ctx(aead
);
573 struct aead_alg
*alg
= crypto_aead_alg(aead
);
574 struct spacc_aead
*spacc_alg
= to_spacc_aead(alg
);
575 struct spacc_engine
*engine
= ctx
->generic
.engine
;
576 u32 ctrl
, proc_len
, assoc_len
;
578 req
->result
= -EINPROGRESS
;
579 req
->ctx_id
= spacc_load_ctx(&ctx
->generic
, ctx
->cipher_key
,
580 ctx
->cipher_key_len
, aead_req
->iv
, crypto_aead_ivsize(aead
),
581 ctx
->hash_ctx
, ctx
->hash_key_len
);
583 /* Set the source and destination DDT pointers. */
584 writel(req
->src_addr
, engine
->regs
+ SPA_SRC_PTR_REG_OFFSET
);
585 writel(req
->dst_addr
, engine
->regs
+ SPA_DST_PTR_REG_OFFSET
);
586 writel(0, engine
->regs
+ SPA_OFFSET_REG_OFFSET
);
588 assoc_len
= aead_req
->assoclen
;
589 proc_len
= aead_req
->cryptlen
+ assoc_len
;
592 * If we are decrypting, we need to take the length of the ICV out of
593 * the processing length.
595 if (!req
->is_encrypt
)
596 proc_len
-= authsize
;
598 writel(proc_len
, engine
->regs
+ SPA_PROC_LEN_REG_OFFSET
);
599 writel(assoc_len
, engine
->regs
+ SPA_AAD_LEN_REG_OFFSET
);
600 writel(authsize
, engine
->regs
+ SPA_ICV_LEN_REG_OFFSET
);
601 writel(0, engine
->regs
+ SPA_ICV_OFFSET_REG_OFFSET
);
602 writel(0, engine
->regs
+ SPA_AUX_INFO_REG_OFFSET
);
604 ctrl
= spacc_alg
->ctrl_default
| (req
->ctx_id
<< SPA_CTRL_CTX_IDX
) |
605 (1 << SPA_CTRL_ICV_APPEND
);
607 ctrl
|= (1 << SPA_CTRL_ENCRYPT_IDX
) | (1 << SPA_CTRL_AAD_COPY
);
609 ctrl
|= (1 << SPA_CTRL_KEY_EXP
);
611 mod_timer(&engine
->packet_timeout
, jiffies
+ PACKET_TIMEOUT
);
613 writel(ctrl
, engine
->regs
+ SPA_CTRL_REG_OFFSET
);
618 static int spacc_req_submit(struct spacc_req
*req
);
620 static void spacc_push(struct spacc_engine
*engine
)
622 struct spacc_req
*req
;
624 while (!list_empty(&engine
->pending
) &&
625 engine
->in_flight
+ 1 <= engine
->fifo_sz
) {
628 req
= list_first_entry(&engine
->pending
, struct spacc_req
,
630 list_move_tail(&req
->list
, &engine
->in_progress
);
632 req
->result
= spacc_req_submit(req
);
637 * Setup an AEAD request for processing. This will configure the engine, load
638 * the context and then start the packet processing.
640 static int spacc_aead_setup(struct aead_request
*req
,
641 unsigned alg_type
, bool is_encrypt
)
643 struct crypto_aead
*aead
= crypto_aead_reqtfm(req
);
644 struct aead_alg
*alg
= crypto_aead_alg(aead
);
645 struct spacc_engine
*engine
= to_spacc_aead(alg
)->engine
;
646 struct spacc_req
*dev_req
= aead_request_ctx(req
);
650 dev_req
->req
= &req
->base
;
651 dev_req
->is_encrypt
= is_encrypt
;
652 dev_req
->result
= -EBUSY
;
653 dev_req
->engine
= engine
;
654 dev_req
->complete
= spacc_aead_complete
;
656 if (unlikely(spacc_aead_need_fallback(req
) ||
657 ((err
= spacc_aead_make_ddts(req
)) == -E2BIG
)))
658 return spacc_aead_do_fallback(req
, alg_type
, is_encrypt
);
664 spin_lock_irqsave(&engine
->hw_lock
, flags
);
665 if (unlikely(spacc_fifo_cmd_full(engine
)) ||
666 engine
->in_flight
+ 1 > engine
->fifo_sz
) {
667 if (!(req
->base
.flags
& CRYPTO_TFM_REQ_MAY_BACKLOG
)) {
669 spin_unlock_irqrestore(&engine
->hw_lock
, flags
);
672 list_add_tail(&dev_req
->list
, &engine
->pending
);
674 list_add_tail(&dev_req
->list
, &engine
->pending
);
677 spin_unlock_irqrestore(&engine
->hw_lock
, flags
);
682 spacc_aead_free_ddts(dev_req
);
687 static int spacc_aead_encrypt(struct aead_request
*req
)
689 struct crypto_aead
*aead
= crypto_aead_reqtfm(req
);
690 struct spacc_aead
*alg
= to_spacc_aead(crypto_aead_alg(aead
));
692 return spacc_aead_setup(req
, alg
->type
, 1);
695 static int spacc_aead_decrypt(struct aead_request
*req
)
697 struct crypto_aead
*aead
= crypto_aead_reqtfm(req
);
698 struct spacc_aead
*alg
= to_spacc_aead(crypto_aead_alg(aead
));
700 return spacc_aead_setup(req
, alg
->type
, 0);
704 * Initialise a new AEAD context. This is responsible for allocating the
705 * fallback cipher and initialising the context.
707 static int spacc_aead_cra_init(struct crypto_aead
*tfm
)
709 struct spacc_aead_ctx
*ctx
= crypto_aead_ctx(tfm
);
710 struct aead_alg
*alg
= crypto_aead_alg(tfm
);
711 struct spacc_aead
*spacc_alg
= to_spacc_aead(alg
);
712 struct spacc_engine
*engine
= spacc_alg
->engine
;
714 ctx
->generic
.flags
= spacc_alg
->type
;
715 ctx
->generic
.engine
= engine
;
716 ctx
->sw_cipher
= crypto_alloc_aead(alg
->base
.cra_name
, 0,
717 CRYPTO_ALG_NEED_FALLBACK
);
718 if (IS_ERR(ctx
->sw_cipher
))
719 return PTR_ERR(ctx
->sw_cipher
);
720 ctx
->generic
.key_offs
= spacc_alg
->key_offs
;
721 ctx
->generic
.iv_offs
= spacc_alg
->iv_offs
;
723 crypto_aead_set_reqsize(
725 max(sizeof(struct spacc_req
),
726 sizeof(struct aead_request
) +
727 crypto_aead_reqsize(ctx
->sw_cipher
)));
733 * Destructor for an AEAD context. This is called when the transform is freed
734 * and must free the fallback cipher.
736 static void spacc_aead_cra_exit(struct crypto_aead
*tfm
)
738 struct spacc_aead_ctx
*ctx
= crypto_aead_ctx(tfm
);
740 crypto_free_aead(ctx
->sw_cipher
);
744 * Set the DES key for a block cipher transform. This also performs weak key
745 * checking if the transform has requested it.
747 static int spacc_des_setkey(struct crypto_ablkcipher
*cipher
, const u8
*key
,
750 struct crypto_tfm
*tfm
= crypto_ablkcipher_tfm(cipher
);
751 struct spacc_ablk_ctx
*ctx
= crypto_tfm_ctx(tfm
);
752 u32 tmp
[DES_EXPKEY_WORDS
];
754 if (len
> DES3_EDE_KEY_SIZE
) {
755 crypto_ablkcipher_set_flags(cipher
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
759 if (unlikely(!des_ekey(tmp
, key
)) &&
760 (crypto_ablkcipher_get_flags(cipher
) & CRYPTO_TFM_REQ_WEAK_KEY
)) {
761 tfm
->crt_flags
|= CRYPTO_TFM_RES_WEAK_KEY
;
765 memcpy(ctx
->key
, key
, len
);
772 * Set the key for an AES block cipher. Some key lengths are not supported in
773 * hardware so this must also check whether a fallback is needed.
775 static int spacc_aes_setkey(struct crypto_ablkcipher
*cipher
, const u8
*key
,
778 struct crypto_tfm
*tfm
= crypto_ablkcipher_tfm(cipher
);
779 struct spacc_ablk_ctx
*ctx
= crypto_tfm_ctx(tfm
);
782 if (len
> AES_MAX_KEY_SIZE
) {
783 crypto_ablkcipher_set_flags(cipher
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
788 * IPSec engine only supports 128 and 256 bit AES keys. If we get a
789 * request for any other size (192 bits) then we need to do a software
792 if (len
!= AES_KEYSIZE_128
&& len
!= AES_KEYSIZE_256
&&
795 * Set the fallback transform to use the same request flags as
796 * the hardware transform.
798 ctx
->sw_cipher
->base
.crt_flags
&= ~CRYPTO_TFM_REQ_MASK
;
799 ctx
->sw_cipher
->base
.crt_flags
|=
800 cipher
->base
.crt_flags
& CRYPTO_TFM_REQ_MASK
;
802 err
= crypto_ablkcipher_setkey(ctx
->sw_cipher
, key
, len
);
804 goto sw_setkey_failed
;
805 } else if (len
!= AES_KEYSIZE_128
&& len
!= AES_KEYSIZE_256
&&
809 memcpy(ctx
->key
, key
, len
);
813 if (err
&& ctx
->sw_cipher
) {
814 tfm
->crt_flags
&= ~CRYPTO_TFM_RES_MASK
;
816 ctx
->sw_cipher
->base
.crt_flags
& CRYPTO_TFM_RES_MASK
;
822 static int spacc_kasumi_f8_setkey(struct crypto_ablkcipher
*cipher
,
823 const u8
*key
, unsigned int len
)
825 struct crypto_tfm
*tfm
= crypto_ablkcipher_tfm(cipher
);
826 struct spacc_ablk_ctx
*ctx
= crypto_tfm_ctx(tfm
);
829 if (len
> AES_MAX_KEY_SIZE
) {
830 crypto_ablkcipher_set_flags(cipher
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
835 memcpy(ctx
->key
, key
, len
);
842 static int spacc_ablk_need_fallback(struct spacc_req
*req
)
844 struct spacc_ablk_ctx
*ctx
;
845 struct crypto_tfm
*tfm
= req
->req
->tfm
;
846 struct crypto_alg
*alg
= req
->req
->tfm
->__crt_alg
;
847 struct spacc_alg
*spacc_alg
= to_spacc_alg(alg
);
849 ctx
= crypto_tfm_ctx(tfm
);
851 return (spacc_alg
->ctrl_default
& SPACC_CRYPTO_ALG_MASK
) ==
852 SPA_CTRL_CIPH_ALG_AES
&&
853 ctx
->key_len
!= AES_KEYSIZE_128
&&
854 ctx
->key_len
!= AES_KEYSIZE_256
;
857 static void spacc_ablk_complete(struct spacc_req
*req
)
859 struct ablkcipher_request
*ablk_req
=
860 container_of(req
->req
, struct ablkcipher_request
, base
);
862 if (ablk_req
->src
!= ablk_req
->dst
) {
863 spacc_free_ddt(req
, req
->src_ddt
, req
->src_addr
, ablk_req
->src
,
864 ablk_req
->nbytes
, DMA_TO_DEVICE
);
865 spacc_free_ddt(req
, req
->dst_ddt
, req
->dst_addr
, ablk_req
->dst
,
866 ablk_req
->nbytes
, DMA_FROM_DEVICE
);
868 spacc_free_ddt(req
, req
->dst_ddt
, req
->dst_addr
, ablk_req
->dst
,
869 ablk_req
->nbytes
, DMA_BIDIRECTIONAL
);
871 req
->req
->complete(req
->req
, req
->result
);
874 static int spacc_ablk_submit(struct spacc_req
*req
)
876 struct crypto_tfm
*tfm
= req
->req
->tfm
;
877 struct spacc_ablk_ctx
*ctx
= crypto_tfm_ctx(tfm
);
878 struct ablkcipher_request
*ablk_req
= ablkcipher_request_cast(req
->req
);
879 struct crypto_alg
*alg
= req
->req
->tfm
->__crt_alg
;
880 struct spacc_alg
*spacc_alg
= to_spacc_alg(alg
);
881 struct spacc_engine
*engine
= ctx
->generic
.engine
;
884 req
->ctx_id
= spacc_load_ctx(&ctx
->generic
, ctx
->key
,
885 ctx
->key_len
, ablk_req
->info
, alg
->cra_ablkcipher
.ivsize
,
888 writel(req
->src_addr
, engine
->regs
+ SPA_SRC_PTR_REG_OFFSET
);
889 writel(req
->dst_addr
, engine
->regs
+ SPA_DST_PTR_REG_OFFSET
);
890 writel(0, engine
->regs
+ SPA_OFFSET_REG_OFFSET
);
892 writel(ablk_req
->nbytes
, engine
->regs
+ SPA_PROC_LEN_REG_OFFSET
);
893 writel(0, engine
->regs
+ SPA_ICV_OFFSET_REG_OFFSET
);
894 writel(0, engine
->regs
+ SPA_AUX_INFO_REG_OFFSET
);
895 writel(0, engine
->regs
+ SPA_AAD_LEN_REG_OFFSET
);
897 ctrl
= spacc_alg
->ctrl_default
| (req
->ctx_id
<< SPA_CTRL_CTX_IDX
) |
898 (req
->is_encrypt
? (1 << SPA_CTRL_ENCRYPT_IDX
) :
899 (1 << SPA_CTRL_KEY_EXP
));
901 mod_timer(&engine
->packet_timeout
, jiffies
+ PACKET_TIMEOUT
);
903 writel(ctrl
, engine
->regs
+ SPA_CTRL_REG_OFFSET
);
908 static int spacc_ablk_do_fallback(struct ablkcipher_request
*req
,
909 unsigned alg_type
, bool is_encrypt
)
911 struct crypto_tfm
*old_tfm
=
912 crypto_ablkcipher_tfm(crypto_ablkcipher_reqtfm(req
));
913 struct spacc_ablk_ctx
*ctx
= crypto_tfm_ctx(old_tfm
);
920 * Change the request to use the software fallback transform, and once
921 * the ciphering has completed, put the old transform back into the
924 ablkcipher_request_set_tfm(req
, ctx
->sw_cipher
);
925 err
= is_encrypt
? crypto_ablkcipher_encrypt(req
) :
926 crypto_ablkcipher_decrypt(req
);
927 ablkcipher_request_set_tfm(req
, __crypto_ablkcipher_cast(old_tfm
));
932 static int spacc_ablk_setup(struct ablkcipher_request
*req
, unsigned alg_type
,
935 struct crypto_alg
*alg
= req
->base
.tfm
->__crt_alg
;
936 struct spacc_engine
*engine
= to_spacc_alg(alg
)->engine
;
937 struct spacc_req
*dev_req
= ablkcipher_request_ctx(req
);
941 dev_req
->req
= &req
->base
;
942 dev_req
->is_encrypt
= is_encrypt
;
943 dev_req
->engine
= engine
;
944 dev_req
->complete
= spacc_ablk_complete
;
945 dev_req
->result
= -EINPROGRESS
;
947 if (unlikely(spacc_ablk_need_fallback(dev_req
)))
948 return spacc_ablk_do_fallback(req
, alg_type
, is_encrypt
);
951 * Create the DDT's for the engine. If we share the same source and
952 * destination then we can optimize by reusing the DDT's.
954 if (req
->src
!= req
->dst
) {
955 dev_req
->src_ddt
= spacc_sg_to_ddt(engine
, req
->src
,
956 req
->nbytes
, DMA_TO_DEVICE
, &dev_req
->src_addr
);
957 if (!dev_req
->src_ddt
)
960 dev_req
->dst_ddt
= spacc_sg_to_ddt(engine
, req
->dst
,
961 req
->nbytes
, DMA_FROM_DEVICE
, &dev_req
->dst_addr
);
962 if (!dev_req
->dst_ddt
)
965 dev_req
->dst_ddt
= spacc_sg_to_ddt(engine
, req
->dst
,
966 req
->nbytes
, DMA_BIDIRECTIONAL
, &dev_req
->dst_addr
);
967 if (!dev_req
->dst_ddt
)
970 dev_req
->src_ddt
= NULL
;
971 dev_req
->src_addr
= dev_req
->dst_addr
;
975 spin_lock_irqsave(&engine
->hw_lock
, flags
);
977 * Check if the engine will accept the operation now. If it won't then
978 * we either stick it on the end of a pending list if we can backlog,
979 * or bailout with an error if not.
981 if (unlikely(spacc_fifo_cmd_full(engine
)) ||
982 engine
->in_flight
+ 1 > engine
->fifo_sz
) {
983 if (!(req
->base
.flags
& CRYPTO_TFM_REQ_MAY_BACKLOG
)) {
985 spin_unlock_irqrestore(&engine
->hw_lock
, flags
);
988 list_add_tail(&dev_req
->list
, &engine
->pending
);
990 list_add_tail(&dev_req
->list
, &engine
->pending
);
993 spin_unlock_irqrestore(&engine
->hw_lock
, flags
);
998 spacc_free_ddt(dev_req
, dev_req
->dst_ddt
, dev_req
->dst_addr
, req
->dst
,
999 req
->nbytes
, req
->src
== req
->dst
?
1000 DMA_BIDIRECTIONAL
: DMA_FROM_DEVICE
);
1002 if (req
->src
!= req
->dst
)
1003 spacc_free_ddt(dev_req
, dev_req
->src_ddt
, dev_req
->src_addr
,
1004 req
->src
, req
->nbytes
, DMA_TO_DEVICE
);
1009 static int spacc_ablk_cra_init(struct crypto_tfm
*tfm
)
1011 struct spacc_ablk_ctx
*ctx
= crypto_tfm_ctx(tfm
);
1012 struct crypto_alg
*alg
= tfm
->__crt_alg
;
1013 struct spacc_alg
*spacc_alg
= to_spacc_alg(alg
);
1014 struct spacc_engine
*engine
= spacc_alg
->engine
;
1016 ctx
->generic
.flags
= spacc_alg
->type
;
1017 ctx
->generic
.engine
= engine
;
1018 if (alg
->cra_flags
& CRYPTO_ALG_NEED_FALLBACK
) {
1019 ctx
->sw_cipher
= crypto_alloc_ablkcipher(alg
->cra_name
, 0,
1020 CRYPTO_ALG_ASYNC
| CRYPTO_ALG_NEED_FALLBACK
);
1021 if (IS_ERR(ctx
->sw_cipher
)) {
1022 dev_warn(engine
->dev
, "failed to allocate fallback for %s\n",
1024 ctx
->sw_cipher
= NULL
;
1027 ctx
->generic
.key_offs
= spacc_alg
->key_offs
;
1028 ctx
->generic
.iv_offs
= spacc_alg
->iv_offs
;
1030 tfm
->crt_ablkcipher
.reqsize
= sizeof(struct spacc_req
);
1035 static void spacc_ablk_cra_exit(struct crypto_tfm
*tfm
)
1037 struct spacc_ablk_ctx
*ctx
= crypto_tfm_ctx(tfm
);
1040 crypto_free_ablkcipher(ctx
->sw_cipher
);
1041 ctx
->sw_cipher
= NULL
;
1044 static int spacc_ablk_encrypt(struct ablkcipher_request
*req
)
1046 struct crypto_ablkcipher
*cipher
= crypto_ablkcipher_reqtfm(req
);
1047 struct crypto_tfm
*tfm
= crypto_ablkcipher_tfm(cipher
);
1048 struct spacc_alg
*alg
= to_spacc_alg(tfm
->__crt_alg
);
1050 return spacc_ablk_setup(req
, alg
->type
, 1);
1053 static int spacc_ablk_decrypt(struct ablkcipher_request
*req
)
1055 struct crypto_ablkcipher
*cipher
= crypto_ablkcipher_reqtfm(req
);
1056 struct crypto_tfm
*tfm
= crypto_ablkcipher_tfm(cipher
);
1057 struct spacc_alg
*alg
= to_spacc_alg(tfm
->__crt_alg
);
1059 return spacc_ablk_setup(req
, alg
->type
, 0);
1062 static inline int spacc_fifo_stat_empty(struct spacc_engine
*engine
)
1064 return readl(engine
->regs
+ SPA_FIFO_STAT_REG_OFFSET
) &
1065 SPA_FIFO_STAT_EMPTY
;
1068 static void spacc_process_done(struct spacc_engine
*engine
)
1070 struct spacc_req
*req
;
1071 unsigned long flags
;
1073 spin_lock_irqsave(&engine
->hw_lock
, flags
);
1075 while (!spacc_fifo_stat_empty(engine
)) {
1076 req
= list_first_entry(&engine
->in_progress
, struct spacc_req
,
1078 list_move_tail(&req
->list
, &engine
->completed
);
1079 --engine
->in_flight
;
1081 /* POP the status register. */
1082 writel(~0, engine
->regs
+ SPA_STAT_POP_REG_OFFSET
);
1083 req
->result
= (readl(engine
->regs
+ SPA_STATUS_REG_OFFSET
) &
1084 SPA_STATUS_RES_CODE_MASK
) >> SPA_STATUS_RES_CODE_OFFSET
;
1087 * Convert the SPAcc error status into the standard POSIX error
1090 if (unlikely(req
->result
)) {
1091 switch (req
->result
) {
1092 case SPA_STATUS_ICV_FAIL
:
1093 req
->result
= -EBADMSG
;
1096 case SPA_STATUS_MEMORY_ERROR
:
1097 dev_warn(engine
->dev
,
1098 "memory error triggered\n");
1099 req
->result
= -EFAULT
;
1102 case SPA_STATUS_BLOCK_ERROR
:
1103 dev_warn(engine
->dev
,
1104 "block error triggered\n");
1111 tasklet_schedule(&engine
->complete
);
1113 spin_unlock_irqrestore(&engine
->hw_lock
, flags
);
1116 static irqreturn_t
spacc_spacc_irq(int irq
, void *dev
)
1118 struct spacc_engine
*engine
= (struct spacc_engine
*)dev
;
1119 u32 spacc_irq_stat
= readl(engine
->regs
+ SPA_IRQ_STAT_REG_OFFSET
);
1121 writel(spacc_irq_stat
, engine
->regs
+ SPA_IRQ_STAT_REG_OFFSET
);
1122 spacc_process_done(engine
);
1127 static void spacc_packet_timeout(unsigned long data
)
1129 struct spacc_engine
*engine
= (struct spacc_engine
*)data
;
1131 spacc_process_done(engine
);
1134 static int spacc_req_submit(struct spacc_req
*req
)
1136 struct crypto_alg
*alg
= req
->req
->tfm
->__crt_alg
;
1138 if (CRYPTO_ALG_TYPE_AEAD
== (CRYPTO_ALG_TYPE_MASK
& alg
->cra_flags
))
1139 return spacc_aead_submit(req
);
1141 return spacc_ablk_submit(req
);
1144 static void spacc_spacc_complete(unsigned long data
)
1146 struct spacc_engine
*engine
= (struct spacc_engine
*)data
;
1147 struct spacc_req
*req
, *tmp
;
1148 unsigned long flags
;
1149 LIST_HEAD(completed
);
1151 spin_lock_irqsave(&engine
->hw_lock
, flags
);
1153 list_splice_init(&engine
->completed
, &completed
);
1155 if (engine
->in_flight
)
1156 mod_timer(&engine
->packet_timeout
, jiffies
+ PACKET_TIMEOUT
);
1158 spin_unlock_irqrestore(&engine
->hw_lock
, flags
);
1160 list_for_each_entry_safe(req
, tmp
, &completed
, list
) {
1161 list_del(&req
->list
);
1167 static int spacc_suspend(struct device
*dev
)
1169 struct platform_device
*pdev
= to_platform_device(dev
);
1170 struct spacc_engine
*engine
= platform_get_drvdata(pdev
);
1173 * We only support standby mode. All we have to do is gate the clock to
1174 * the spacc. The hardware will preserve state until we turn it back
1177 clk_disable(engine
->clk
);
1182 static int spacc_resume(struct device
*dev
)
1184 struct platform_device
*pdev
= to_platform_device(dev
);
1185 struct spacc_engine
*engine
= platform_get_drvdata(pdev
);
1187 return clk_enable(engine
->clk
);
1190 static const struct dev_pm_ops spacc_pm_ops
= {
1191 .suspend
= spacc_suspend
,
1192 .resume
= spacc_resume
,
1194 #endif /* CONFIG_PM */
1196 static inline struct spacc_engine
*spacc_dev_to_engine(struct device
*dev
)
1198 return dev
? platform_get_drvdata(to_platform_device(dev
)) : NULL
;
1201 static ssize_t
spacc_stat_irq_thresh_show(struct device
*dev
,
1202 struct device_attribute
*attr
,
1205 struct spacc_engine
*engine
= spacc_dev_to_engine(dev
);
1207 return snprintf(buf
, PAGE_SIZE
, "%u\n", engine
->stat_irq_thresh
);
1210 static ssize_t
spacc_stat_irq_thresh_store(struct device
*dev
,
1211 struct device_attribute
*attr
,
1212 const char *buf
, size_t len
)
1214 struct spacc_engine
*engine
= spacc_dev_to_engine(dev
);
1215 unsigned long thresh
;
1217 if (kstrtoul(buf
, 0, &thresh
))
1220 thresh
= clamp(thresh
, 1UL, engine
->fifo_sz
- 1);
1222 engine
->stat_irq_thresh
= thresh
;
1223 writel(engine
->stat_irq_thresh
<< SPA_IRQ_CTRL_STAT_CNT_OFFSET
,
1224 engine
->regs
+ SPA_IRQ_CTRL_REG_OFFSET
);
1228 static DEVICE_ATTR(stat_irq_thresh
, 0644, spacc_stat_irq_thresh_show
,
1229 spacc_stat_irq_thresh_store
);
1231 static struct spacc_alg ipsec_engine_algs
[] = {
1233 .ctrl_default
= SPA_CTRL_CIPH_ALG_AES
| SPA_CTRL_CIPH_MODE_CBC
,
1235 .iv_offs
= AES_MAX_KEY_SIZE
,
1237 .cra_name
= "cbc(aes)",
1238 .cra_driver_name
= "cbc-aes-picoxcell",
1239 .cra_priority
= SPACC_CRYPTO_ALG_PRIORITY
,
1240 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|
1241 CRYPTO_ALG_KERN_DRIVER_ONLY
|
1243 CRYPTO_ALG_NEED_FALLBACK
,
1244 .cra_blocksize
= AES_BLOCK_SIZE
,
1245 .cra_ctxsize
= sizeof(struct spacc_ablk_ctx
),
1246 .cra_type
= &crypto_ablkcipher_type
,
1247 .cra_module
= THIS_MODULE
,
1249 .setkey
= spacc_aes_setkey
,
1250 .encrypt
= spacc_ablk_encrypt
,
1251 .decrypt
= spacc_ablk_decrypt
,
1252 .min_keysize
= AES_MIN_KEY_SIZE
,
1253 .max_keysize
= AES_MAX_KEY_SIZE
,
1254 .ivsize
= AES_BLOCK_SIZE
,
1256 .cra_init
= spacc_ablk_cra_init
,
1257 .cra_exit
= spacc_ablk_cra_exit
,
1262 .iv_offs
= AES_MAX_KEY_SIZE
,
1263 .ctrl_default
= SPA_CTRL_CIPH_ALG_AES
| SPA_CTRL_CIPH_MODE_ECB
,
1265 .cra_name
= "ecb(aes)",
1266 .cra_driver_name
= "ecb-aes-picoxcell",
1267 .cra_priority
= SPACC_CRYPTO_ALG_PRIORITY
,
1268 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|
1269 CRYPTO_ALG_KERN_DRIVER_ONLY
|
1270 CRYPTO_ALG_ASYNC
| CRYPTO_ALG_NEED_FALLBACK
,
1271 .cra_blocksize
= AES_BLOCK_SIZE
,
1272 .cra_ctxsize
= sizeof(struct spacc_ablk_ctx
),
1273 .cra_type
= &crypto_ablkcipher_type
,
1274 .cra_module
= THIS_MODULE
,
1276 .setkey
= spacc_aes_setkey
,
1277 .encrypt
= spacc_ablk_encrypt
,
1278 .decrypt
= spacc_ablk_decrypt
,
1279 .min_keysize
= AES_MIN_KEY_SIZE
,
1280 .max_keysize
= AES_MAX_KEY_SIZE
,
1282 .cra_init
= spacc_ablk_cra_init
,
1283 .cra_exit
= spacc_ablk_cra_exit
,
1287 .key_offs
= DES_BLOCK_SIZE
,
1289 .ctrl_default
= SPA_CTRL_CIPH_ALG_DES
| SPA_CTRL_CIPH_MODE_CBC
,
1291 .cra_name
= "cbc(des)",
1292 .cra_driver_name
= "cbc-des-picoxcell",
1293 .cra_priority
= SPACC_CRYPTO_ALG_PRIORITY
,
1294 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|
1296 CRYPTO_ALG_KERN_DRIVER_ONLY
,
1297 .cra_blocksize
= DES_BLOCK_SIZE
,
1298 .cra_ctxsize
= sizeof(struct spacc_ablk_ctx
),
1299 .cra_type
= &crypto_ablkcipher_type
,
1300 .cra_module
= THIS_MODULE
,
1302 .setkey
= spacc_des_setkey
,
1303 .encrypt
= spacc_ablk_encrypt
,
1304 .decrypt
= spacc_ablk_decrypt
,
1305 .min_keysize
= DES_KEY_SIZE
,
1306 .max_keysize
= DES_KEY_SIZE
,
1307 .ivsize
= DES_BLOCK_SIZE
,
1309 .cra_init
= spacc_ablk_cra_init
,
1310 .cra_exit
= spacc_ablk_cra_exit
,
1314 .key_offs
= DES_BLOCK_SIZE
,
1316 .ctrl_default
= SPA_CTRL_CIPH_ALG_DES
| SPA_CTRL_CIPH_MODE_ECB
,
1318 .cra_name
= "ecb(des)",
1319 .cra_driver_name
= "ecb-des-picoxcell",
1320 .cra_priority
= SPACC_CRYPTO_ALG_PRIORITY
,
1321 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|
1323 CRYPTO_ALG_KERN_DRIVER_ONLY
,
1324 .cra_blocksize
= DES_BLOCK_SIZE
,
1325 .cra_ctxsize
= sizeof(struct spacc_ablk_ctx
),
1326 .cra_type
= &crypto_ablkcipher_type
,
1327 .cra_module
= THIS_MODULE
,
1329 .setkey
= spacc_des_setkey
,
1330 .encrypt
= spacc_ablk_encrypt
,
1331 .decrypt
= spacc_ablk_decrypt
,
1332 .min_keysize
= DES_KEY_SIZE
,
1333 .max_keysize
= DES_KEY_SIZE
,
1335 .cra_init
= spacc_ablk_cra_init
,
1336 .cra_exit
= spacc_ablk_cra_exit
,
1340 .key_offs
= DES_BLOCK_SIZE
,
1342 .ctrl_default
= SPA_CTRL_CIPH_ALG_DES
| SPA_CTRL_CIPH_MODE_CBC
,
1344 .cra_name
= "cbc(des3_ede)",
1345 .cra_driver_name
= "cbc-des3-ede-picoxcell",
1346 .cra_priority
= SPACC_CRYPTO_ALG_PRIORITY
,
1347 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|
1349 CRYPTO_ALG_KERN_DRIVER_ONLY
,
1350 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
1351 .cra_ctxsize
= sizeof(struct spacc_ablk_ctx
),
1352 .cra_type
= &crypto_ablkcipher_type
,
1353 .cra_module
= THIS_MODULE
,
1355 .setkey
= spacc_des_setkey
,
1356 .encrypt
= spacc_ablk_encrypt
,
1357 .decrypt
= spacc_ablk_decrypt
,
1358 .min_keysize
= DES3_EDE_KEY_SIZE
,
1359 .max_keysize
= DES3_EDE_KEY_SIZE
,
1360 .ivsize
= DES3_EDE_BLOCK_SIZE
,
1362 .cra_init
= spacc_ablk_cra_init
,
1363 .cra_exit
= spacc_ablk_cra_exit
,
1367 .key_offs
= DES_BLOCK_SIZE
,
1369 .ctrl_default
= SPA_CTRL_CIPH_ALG_DES
| SPA_CTRL_CIPH_MODE_ECB
,
1371 .cra_name
= "ecb(des3_ede)",
1372 .cra_driver_name
= "ecb-des3-ede-picoxcell",
1373 .cra_priority
= SPACC_CRYPTO_ALG_PRIORITY
,
1374 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|
1376 CRYPTO_ALG_KERN_DRIVER_ONLY
,
1377 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
1378 .cra_ctxsize
= sizeof(struct spacc_ablk_ctx
),
1379 .cra_type
= &crypto_ablkcipher_type
,
1380 .cra_module
= THIS_MODULE
,
1382 .setkey
= spacc_des_setkey
,
1383 .encrypt
= spacc_ablk_encrypt
,
1384 .decrypt
= spacc_ablk_decrypt
,
1385 .min_keysize
= DES3_EDE_KEY_SIZE
,
1386 .max_keysize
= DES3_EDE_KEY_SIZE
,
1388 .cra_init
= spacc_ablk_cra_init
,
1389 .cra_exit
= spacc_ablk_cra_exit
,
1394 static struct spacc_aead ipsec_engine_aeads
[] = {
1396 .ctrl_default
= SPA_CTRL_CIPH_ALG_AES
|
1397 SPA_CTRL_CIPH_MODE_CBC
|
1398 SPA_CTRL_HASH_ALG_SHA
|
1399 SPA_CTRL_HASH_MODE_HMAC
,
1401 .iv_offs
= AES_MAX_KEY_SIZE
,
1404 .cra_name
= "authenc(hmac(sha1),cbc(aes))",
1405 .cra_driver_name
= "authenc-hmac-sha1-"
1406 "cbc-aes-picoxcell",
1407 .cra_priority
= SPACC_CRYPTO_ALG_PRIORITY
,
1408 .cra_flags
= CRYPTO_ALG_ASYNC
|
1409 CRYPTO_ALG_NEED_FALLBACK
|
1410 CRYPTO_ALG_KERN_DRIVER_ONLY
,
1411 .cra_blocksize
= AES_BLOCK_SIZE
,
1412 .cra_ctxsize
= sizeof(struct spacc_aead_ctx
),
1413 .cra_module
= THIS_MODULE
,
1415 .setkey
= spacc_aead_setkey
,
1416 .setauthsize
= spacc_aead_setauthsize
,
1417 .encrypt
= spacc_aead_encrypt
,
1418 .decrypt
= spacc_aead_decrypt
,
1419 .ivsize
= AES_BLOCK_SIZE
,
1420 .maxauthsize
= SHA1_DIGEST_SIZE
,
1421 .init
= spacc_aead_cra_init
,
1422 .exit
= spacc_aead_cra_exit
,
1426 .ctrl_default
= SPA_CTRL_CIPH_ALG_AES
|
1427 SPA_CTRL_CIPH_MODE_CBC
|
1428 SPA_CTRL_HASH_ALG_SHA256
|
1429 SPA_CTRL_HASH_MODE_HMAC
,
1431 .iv_offs
= AES_MAX_KEY_SIZE
,
1434 .cra_name
= "authenc(hmac(sha256),cbc(aes))",
1435 .cra_driver_name
= "authenc-hmac-sha256-"
1436 "cbc-aes-picoxcell",
1437 .cra_priority
= SPACC_CRYPTO_ALG_PRIORITY
,
1438 .cra_flags
= CRYPTO_ALG_ASYNC
|
1439 CRYPTO_ALG_NEED_FALLBACK
|
1440 CRYPTO_ALG_KERN_DRIVER_ONLY
,
1441 .cra_blocksize
= AES_BLOCK_SIZE
,
1442 .cra_ctxsize
= sizeof(struct spacc_aead_ctx
),
1443 .cra_module
= THIS_MODULE
,
1445 .setkey
= spacc_aead_setkey
,
1446 .setauthsize
= spacc_aead_setauthsize
,
1447 .encrypt
= spacc_aead_encrypt
,
1448 .decrypt
= spacc_aead_decrypt
,
1449 .ivsize
= AES_BLOCK_SIZE
,
1450 .maxauthsize
= SHA256_DIGEST_SIZE
,
1451 .init
= spacc_aead_cra_init
,
1452 .exit
= spacc_aead_cra_exit
,
1457 .iv_offs
= AES_MAX_KEY_SIZE
,
1458 .ctrl_default
= SPA_CTRL_CIPH_ALG_AES
|
1459 SPA_CTRL_CIPH_MODE_CBC
|
1460 SPA_CTRL_HASH_ALG_MD5
|
1461 SPA_CTRL_HASH_MODE_HMAC
,
1464 .cra_name
= "authenc(hmac(md5),cbc(aes))",
1465 .cra_driver_name
= "authenc-hmac-md5-"
1466 "cbc-aes-picoxcell",
1467 .cra_priority
= SPACC_CRYPTO_ALG_PRIORITY
,
1468 .cra_flags
= CRYPTO_ALG_ASYNC
|
1469 CRYPTO_ALG_NEED_FALLBACK
|
1470 CRYPTO_ALG_KERN_DRIVER_ONLY
,
1471 .cra_blocksize
= AES_BLOCK_SIZE
,
1472 .cra_ctxsize
= sizeof(struct spacc_aead_ctx
),
1473 .cra_module
= THIS_MODULE
,
1475 .setkey
= spacc_aead_setkey
,
1476 .setauthsize
= spacc_aead_setauthsize
,
1477 .encrypt
= spacc_aead_encrypt
,
1478 .decrypt
= spacc_aead_decrypt
,
1479 .ivsize
= AES_BLOCK_SIZE
,
1480 .maxauthsize
= MD5_DIGEST_SIZE
,
1481 .init
= spacc_aead_cra_init
,
1482 .exit
= spacc_aead_cra_exit
,
1486 .key_offs
= DES_BLOCK_SIZE
,
1488 .ctrl_default
= SPA_CTRL_CIPH_ALG_DES
|
1489 SPA_CTRL_CIPH_MODE_CBC
|
1490 SPA_CTRL_HASH_ALG_SHA
|
1491 SPA_CTRL_HASH_MODE_HMAC
,
1494 .cra_name
= "authenc(hmac(sha1),cbc(des3_ede))",
1495 .cra_driver_name
= "authenc-hmac-sha1-"
1496 "cbc-3des-picoxcell",
1497 .cra_priority
= SPACC_CRYPTO_ALG_PRIORITY
,
1498 .cra_flags
= CRYPTO_ALG_ASYNC
|
1499 CRYPTO_ALG_NEED_FALLBACK
|
1500 CRYPTO_ALG_KERN_DRIVER_ONLY
,
1501 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
1502 .cra_ctxsize
= sizeof(struct spacc_aead_ctx
),
1503 .cra_module
= THIS_MODULE
,
1505 .setkey
= spacc_aead_setkey
,
1506 .setauthsize
= spacc_aead_setauthsize
,
1507 .encrypt
= spacc_aead_encrypt
,
1508 .decrypt
= spacc_aead_decrypt
,
1509 .ivsize
= DES3_EDE_BLOCK_SIZE
,
1510 .maxauthsize
= SHA1_DIGEST_SIZE
,
1511 .init
= spacc_aead_cra_init
,
1512 .exit
= spacc_aead_cra_exit
,
1516 .key_offs
= DES_BLOCK_SIZE
,
1518 .ctrl_default
= SPA_CTRL_CIPH_ALG_AES
|
1519 SPA_CTRL_CIPH_MODE_CBC
|
1520 SPA_CTRL_HASH_ALG_SHA256
|
1521 SPA_CTRL_HASH_MODE_HMAC
,
1524 .cra_name
= "authenc(hmac(sha256),"
1526 .cra_driver_name
= "authenc-hmac-sha256-"
1527 "cbc-3des-picoxcell",
1528 .cra_priority
= SPACC_CRYPTO_ALG_PRIORITY
,
1529 .cra_flags
= CRYPTO_ALG_ASYNC
|
1530 CRYPTO_ALG_NEED_FALLBACK
|
1531 CRYPTO_ALG_KERN_DRIVER_ONLY
,
1532 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
1533 .cra_ctxsize
= sizeof(struct spacc_aead_ctx
),
1534 .cra_module
= THIS_MODULE
,
1536 .setkey
= spacc_aead_setkey
,
1537 .setauthsize
= spacc_aead_setauthsize
,
1538 .encrypt
= spacc_aead_encrypt
,
1539 .decrypt
= spacc_aead_decrypt
,
1540 .ivsize
= DES3_EDE_BLOCK_SIZE
,
1541 .maxauthsize
= SHA256_DIGEST_SIZE
,
1542 .init
= spacc_aead_cra_init
,
1543 .exit
= spacc_aead_cra_exit
,
1547 .key_offs
= DES_BLOCK_SIZE
,
1549 .ctrl_default
= SPA_CTRL_CIPH_ALG_DES
|
1550 SPA_CTRL_CIPH_MODE_CBC
|
1551 SPA_CTRL_HASH_ALG_MD5
|
1552 SPA_CTRL_HASH_MODE_HMAC
,
1555 .cra_name
= "authenc(hmac(md5),cbc(des3_ede))",
1556 .cra_driver_name
= "authenc-hmac-md5-"
1557 "cbc-3des-picoxcell",
1558 .cra_priority
= SPACC_CRYPTO_ALG_PRIORITY
,
1559 .cra_flags
= CRYPTO_ALG_ASYNC
|
1560 CRYPTO_ALG_NEED_FALLBACK
|
1561 CRYPTO_ALG_KERN_DRIVER_ONLY
,
1562 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
1563 .cra_ctxsize
= sizeof(struct spacc_aead_ctx
),
1564 .cra_module
= THIS_MODULE
,
1566 .setkey
= spacc_aead_setkey
,
1567 .setauthsize
= spacc_aead_setauthsize
,
1568 .encrypt
= spacc_aead_encrypt
,
1569 .decrypt
= spacc_aead_decrypt
,
1570 .ivsize
= DES3_EDE_BLOCK_SIZE
,
1571 .maxauthsize
= MD5_DIGEST_SIZE
,
1572 .init
= spacc_aead_cra_init
,
1573 .exit
= spacc_aead_cra_exit
,
1578 static struct spacc_alg l2_engine_algs
[] = {
1581 .iv_offs
= SPACC_CRYPTO_KASUMI_F8_KEY_LEN
,
1582 .ctrl_default
= SPA_CTRL_CIPH_ALG_KASUMI
|
1583 SPA_CTRL_CIPH_MODE_F8
,
1585 .cra_name
= "f8(kasumi)",
1586 .cra_driver_name
= "f8-kasumi-picoxcell",
1587 .cra_priority
= SPACC_CRYPTO_ALG_PRIORITY
,
1588 .cra_flags
= CRYPTO_ALG_TYPE_GIVCIPHER
|
1590 CRYPTO_ALG_KERN_DRIVER_ONLY
,
1592 .cra_ctxsize
= sizeof(struct spacc_ablk_ctx
),
1593 .cra_type
= &crypto_ablkcipher_type
,
1594 .cra_module
= THIS_MODULE
,
1596 .setkey
= spacc_kasumi_f8_setkey
,
1597 .encrypt
= spacc_ablk_encrypt
,
1598 .decrypt
= spacc_ablk_decrypt
,
1603 .cra_init
= spacc_ablk_cra_init
,
1604 .cra_exit
= spacc_ablk_cra_exit
,
1610 static const struct of_device_id spacc_of_id_table
[] = {
1611 { .compatible
= "picochip,spacc-ipsec" },
1612 { .compatible
= "picochip,spacc-l2" },
1615 MODULE_DEVICE_TABLE(of
, spacc_of_id_table
);
1616 #endif /* CONFIG_OF */
1618 static bool spacc_is_compatible(struct platform_device
*pdev
,
1619 const char *spacc_type
)
1621 const struct platform_device_id
*platid
= platform_get_device_id(pdev
);
1623 if (platid
&& !strcmp(platid
->name
, spacc_type
))
1627 if (of_device_is_compatible(pdev
->dev
.of_node
, spacc_type
))
1629 #endif /* CONFIG_OF */
1634 static int spacc_probe(struct platform_device
*pdev
)
1636 int i
, err
, ret
= -EINVAL
;
1637 struct resource
*mem
, *irq
;
1638 struct spacc_engine
*engine
= devm_kzalloc(&pdev
->dev
, sizeof(*engine
),
1643 if (spacc_is_compatible(pdev
, "picochip,spacc-ipsec")) {
1644 engine
->max_ctxs
= SPACC_CRYPTO_IPSEC_MAX_CTXS
;
1645 engine
->cipher_pg_sz
= SPACC_CRYPTO_IPSEC_CIPHER_PG_SZ
;
1646 engine
->hash_pg_sz
= SPACC_CRYPTO_IPSEC_HASH_PG_SZ
;
1647 engine
->fifo_sz
= SPACC_CRYPTO_IPSEC_FIFO_SZ
;
1648 engine
->algs
= ipsec_engine_algs
;
1649 engine
->num_algs
= ARRAY_SIZE(ipsec_engine_algs
);
1650 engine
->aeads
= ipsec_engine_aeads
;
1651 engine
->num_aeads
= ARRAY_SIZE(ipsec_engine_aeads
);
1652 } else if (spacc_is_compatible(pdev
, "picochip,spacc-l2")) {
1653 engine
->max_ctxs
= SPACC_CRYPTO_L2_MAX_CTXS
;
1654 engine
->cipher_pg_sz
= SPACC_CRYPTO_L2_CIPHER_PG_SZ
;
1655 engine
->hash_pg_sz
= SPACC_CRYPTO_L2_HASH_PG_SZ
;
1656 engine
->fifo_sz
= SPACC_CRYPTO_L2_FIFO_SZ
;
1657 engine
->algs
= l2_engine_algs
;
1658 engine
->num_algs
= ARRAY_SIZE(l2_engine_algs
);
1663 engine
->name
= dev_name(&pdev
->dev
);
1665 mem
= platform_get_resource(pdev
, IORESOURCE_MEM
, 0);
1666 engine
->regs
= devm_ioremap_resource(&pdev
->dev
, mem
);
1667 if (IS_ERR(engine
->regs
))
1668 return PTR_ERR(engine
->regs
);
1670 irq
= platform_get_resource(pdev
, IORESOURCE_IRQ
, 0);
1672 dev_err(&pdev
->dev
, "no memory/irq resource for engine\n");
1676 if (devm_request_irq(&pdev
->dev
, irq
->start
, spacc_spacc_irq
, 0,
1677 engine
->name
, engine
)) {
1678 dev_err(engine
->dev
, "failed to request IRQ\n");
1682 engine
->dev
= &pdev
->dev
;
1683 engine
->cipher_ctx_base
= engine
->regs
+ SPA_CIPH_KEY_BASE_REG_OFFSET
;
1684 engine
->hash_key_base
= engine
->regs
+ SPA_HASH_KEY_BASE_REG_OFFSET
;
1686 engine
->req_pool
= dmam_pool_create(engine
->name
, engine
->dev
,
1687 MAX_DDT_LEN
* sizeof(struct spacc_ddt
), 8, SZ_64K
);
1688 if (!engine
->req_pool
)
1691 spin_lock_init(&engine
->hw_lock
);
1693 engine
->clk
= clk_get(&pdev
->dev
, "ref");
1694 if (IS_ERR(engine
->clk
)) {
1695 dev_info(&pdev
->dev
, "clk unavailable\n");
1696 device_remove_file(&pdev
->dev
, &dev_attr_stat_irq_thresh
);
1697 return PTR_ERR(engine
->clk
);
1700 if (clk_prepare_enable(engine
->clk
)) {
1701 dev_info(&pdev
->dev
, "unable to prepare/enable clk\n");
1702 clk_put(engine
->clk
);
1706 err
= device_create_file(&pdev
->dev
, &dev_attr_stat_irq_thresh
);
1708 clk_disable_unprepare(engine
->clk
);
1709 clk_put(engine
->clk
);
1715 * Use an IRQ threshold of 50% as a default. This seems to be a
1716 * reasonable trade off of latency against throughput but can be
1717 * changed at runtime.
1719 engine
->stat_irq_thresh
= (engine
->fifo_sz
/ 2);
1722 * Configure the interrupts. We only use the STAT_CNT interrupt as we
1723 * only submit a new packet for processing when we complete another in
1724 * the queue. This minimizes time spent in the interrupt handler.
1726 writel(engine
->stat_irq_thresh
<< SPA_IRQ_CTRL_STAT_CNT_OFFSET
,
1727 engine
->regs
+ SPA_IRQ_CTRL_REG_OFFSET
);
1728 writel(SPA_IRQ_EN_STAT_EN
| SPA_IRQ_EN_GLBL_EN
,
1729 engine
->regs
+ SPA_IRQ_EN_REG_OFFSET
);
1731 setup_timer(&engine
->packet_timeout
, spacc_packet_timeout
,
1732 (unsigned long)engine
);
1734 INIT_LIST_HEAD(&engine
->pending
);
1735 INIT_LIST_HEAD(&engine
->completed
);
1736 INIT_LIST_HEAD(&engine
->in_progress
);
1737 engine
->in_flight
= 0;
1738 tasklet_init(&engine
->complete
, spacc_spacc_complete
,
1739 (unsigned long)engine
);
1741 platform_set_drvdata(pdev
, engine
);
1743 INIT_LIST_HEAD(&engine
->registered_algs
);
1744 for (i
= 0; i
< engine
->num_algs
; ++i
) {
1745 engine
->algs
[i
].engine
= engine
;
1746 err
= crypto_register_alg(&engine
->algs
[i
].alg
);
1748 list_add_tail(&engine
->algs
[i
].entry
,
1749 &engine
->registered_algs
);
1753 dev_err(engine
->dev
, "failed to register alg \"%s\"\n",
1754 engine
->algs
[i
].alg
.cra_name
);
1756 dev_dbg(engine
->dev
, "registered alg \"%s\"\n",
1757 engine
->algs
[i
].alg
.cra_name
);
1760 INIT_LIST_HEAD(&engine
->registered_aeads
);
1761 for (i
= 0; i
< engine
->num_aeads
; ++i
) {
1762 engine
->aeads
[i
].engine
= engine
;
1763 err
= crypto_register_aead(&engine
->aeads
[i
].alg
);
1765 list_add_tail(&engine
->aeads
[i
].entry
,
1766 &engine
->registered_aeads
);
1770 dev_err(engine
->dev
, "failed to register alg \"%s\"\n",
1771 engine
->aeads
[i
].alg
.base
.cra_name
);
1773 dev_dbg(engine
->dev
, "registered alg \"%s\"\n",
1774 engine
->aeads
[i
].alg
.base
.cra_name
);
1780 static int spacc_remove(struct platform_device
*pdev
)
1782 struct spacc_aead
*aead
, *an
;
1783 struct spacc_alg
*alg
, *next
;
1784 struct spacc_engine
*engine
= platform_get_drvdata(pdev
);
1786 del_timer_sync(&engine
->packet_timeout
);
1787 device_remove_file(&pdev
->dev
, &dev_attr_stat_irq_thresh
);
1789 list_for_each_entry_safe(aead
, an
, &engine
->registered_aeads
, entry
) {
1790 list_del(&aead
->entry
);
1791 crypto_unregister_aead(&aead
->alg
);
1794 list_for_each_entry_safe(alg
, next
, &engine
->registered_algs
, entry
) {
1795 list_del(&alg
->entry
);
1796 crypto_unregister_alg(&alg
->alg
);
1799 clk_disable_unprepare(engine
->clk
);
1800 clk_put(engine
->clk
);
1805 static const struct platform_device_id spacc_id_table
[] = {
1806 { "picochip,spacc-ipsec", },
1807 { "picochip,spacc-l2", },
1811 static struct platform_driver spacc_driver
= {
1812 .probe
= spacc_probe
,
1813 .remove
= spacc_remove
,
1815 .name
= "picochip,spacc",
1817 .pm
= &spacc_pm_ops
,
1818 #endif /* CONFIG_PM */
1819 .of_match_table
= of_match_ptr(spacc_of_id_table
),
1821 .id_table
= spacc_id_table
,
1824 module_platform_driver(spacc_driver
);
1826 MODULE_LICENSE("GPL");
1827 MODULE_AUTHOR("Jamie Iles");