2 * talitos - Freescale Integrated Security Engine (SEC) device driver
4 * Copyright (c) 2008-2011 Freescale Semiconductor, Inc.
6 * Scatterlist Crypto API glue code copied from files with the following:
7 * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
9 * Crypto algorithm registration code copied from hifn driver:
10 * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
11 * All rights reserved.
13 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
23 * You should have received a copy of the GNU General Public License
24 * along with this program; if not, write to the Free Software
25 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
28 #include <linux/kernel.h>
29 #include <linux/module.h>
30 #include <linux/mod_devicetable.h>
31 #include <linux/device.h>
32 #include <linux/interrupt.h>
33 #include <linux/crypto.h>
34 #include <linux/hw_random.h>
35 #include <linux/of_address.h>
36 #include <linux/of_irq.h>
37 #include <linux/of_platform.h>
38 #include <linux/dma-mapping.h>
40 #include <linux/spinlock.h>
41 #include <linux/rtnetlink.h>
42 #include <linux/slab.h>
44 #include <crypto/algapi.h>
45 #include <crypto/aes.h>
46 #include <crypto/des.h>
47 #include <crypto/sha.h>
48 #include <crypto/md5.h>
49 #include <crypto/aead.h>
50 #include <crypto/authenc.h>
51 #include <crypto/skcipher.h>
52 #include <crypto/hash.h>
53 #include <crypto/internal/hash.h>
54 #include <crypto/scatterwalk.h>
58 static void to_talitos_ptr(struct talitos_ptr
*ptr
, dma_addr_t dma_addr
,
61 ptr
->ptr
= cpu_to_be32(lower_32_bits(dma_addr
));
63 ptr
->eptr
= upper_32_bits(dma_addr
);
66 static void to_talitos_ptr_len(struct talitos_ptr
*ptr
, unsigned short len
,
71 ptr
->len1
= cpu_to_be16(len
);
73 ptr
->len
= cpu_to_be16(len
);
77 static unsigned short from_talitos_ptr_len(struct talitos_ptr
*ptr
,
81 return be16_to_cpu(ptr
->len1
);
83 return be16_to_cpu(ptr
->len
);
86 static void to_talitos_ptr_extent_clear(struct talitos_ptr
*ptr
, bool is_sec1
)
93 * map virtual single (contiguous) pointer to h/w descriptor pointer
95 static void map_single_talitos_ptr(struct device
*dev
,
96 struct talitos_ptr
*ptr
,
97 unsigned short len
, void *data
,
98 enum dma_data_direction dir
)
100 dma_addr_t dma_addr
= dma_map_single(dev
, data
, len
, dir
);
101 struct talitos_private
*priv
= dev_get_drvdata(dev
);
102 bool is_sec1
= has_ftr_sec1(priv
);
104 to_talitos_ptr_len(ptr
, len
, is_sec1
);
105 to_talitos_ptr(ptr
, dma_addr
, is_sec1
);
106 to_talitos_ptr_extent_clear(ptr
, is_sec1
);
110 * unmap bus single (contiguous) h/w descriptor pointer
112 static void unmap_single_talitos_ptr(struct device
*dev
,
113 struct talitos_ptr
*ptr
,
114 enum dma_data_direction dir
)
116 struct talitos_private
*priv
= dev_get_drvdata(dev
);
117 bool is_sec1
= has_ftr_sec1(priv
);
119 dma_unmap_single(dev
, be32_to_cpu(ptr
->ptr
),
120 from_talitos_ptr_len(ptr
, is_sec1
), dir
);
123 static int reset_channel(struct device
*dev
, int ch
)
125 struct talitos_private
*priv
= dev_get_drvdata(dev
);
126 unsigned int timeout
= TALITOS_TIMEOUT
;
128 setbits32(priv
->chan
[ch
].reg
+ TALITOS_CCCR
, TALITOS_CCCR_RESET
);
130 while ((in_be32(priv
->chan
[ch
].reg
+ TALITOS_CCCR
) & TALITOS_CCCR_RESET
)
135 dev_err(dev
, "failed to reset channel %d\n", ch
);
139 /* set 36-bit addressing, done writeback enable and done IRQ enable */
140 setbits32(priv
->chan
[ch
].reg
+ TALITOS_CCCR_LO
, TALITOS_CCCR_LO_EAE
|
141 TALITOS_CCCR_LO_CDWE
| TALITOS_CCCR_LO_CDIE
);
143 /* and ICCR writeback, if available */
144 if (priv
->features
& TALITOS_FTR_HW_AUTH_CHECK
)
145 setbits32(priv
->chan
[ch
].reg
+ TALITOS_CCCR_LO
,
146 TALITOS_CCCR_LO_IWSE
);
151 static int reset_device(struct device
*dev
)
153 struct talitos_private
*priv
= dev_get_drvdata(dev
);
154 unsigned int timeout
= TALITOS_TIMEOUT
;
155 u32 mcr
= TALITOS_MCR_SWR
;
157 setbits32(priv
->reg
+ TALITOS_MCR
, mcr
);
159 while ((in_be32(priv
->reg
+ TALITOS_MCR
) & TALITOS_MCR_SWR
)
164 mcr
= TALITOS_MCR_RCA1
| TALITOS_MCR_RCA3
;
165 setbits32(priv
->reg
+ TALITOS_MCR
, mcr
);
169 dev_err(dev
, "failed to reset device\n");
177 * Reset and initialize the device
179 static int init_device(struct device
*dev
)
181 struct talitos_private
*priv
= dev_get_drvdata(dev
);
186 * errata documentation: warning: certain SEC interrupts
187 * are not fully cleared by writing the MCR:SWR bit,
188 * set bit twice to completely reset
190 err
= reset_device(dev
);
194 err
= reset_device(dev
);
199 for (ch
= 0; ch
< priv
->num_channels
; ch
++) {
200 err
= reset_channel(dev
, ch
);
205 /* enable channel done and error interrupts */
206 setbits32(priv
->reg
+ TALITOS_IMR
, TALITOS_IMR_INIT
);
207 setbits32(priv
->reg
+ TALITOS_IMR_LO
, TALITOS_IMR_LO_INIT
);
209 /* disable integrity check error interrupts (use writeback instead) */
210 if (priv
->features
& TALITOS_FTR_HW_AUTH_CHECK
)
211 setbits32(priv
->reg
+ TALITOS_MDEUICR_LO
,
212 TALITOS_MDEUICR_LO_ICE
);
218 * talitos_submit - submits a descriptor to the device for processing
219 * @dev: the SEC device to be used
220 * @ch: the SEC device channel to be used
221 * @desc: the descriptor to be processed by the device
222 * @callback: whom to call when processing is complete
223 * @context: a handle for use by caller (optional)
225 * desc must contain valid dma-mapped (bus physical) address pointers.
226 * callback must check err and feedback in descriptor header
227 * for device processing status.
229 int talitos_submit(struct device
*dev
, int ch
, struct talitos_desc
*desc
,
230 void (*callback
)(struct device
*dev
,
231 struct talitos_desc
*desc
,
232 void *context
, int error
),
235 struct talitos_private
*priv
= dev_get_drvdata(dev
);
236 struct talitos_request
*request
;
239 bool is_sec1
= has_ftr_sec1(priv
);
241 spin_lock_irqsave(&priv
->chan
[ch
].head_lock
, flags
);
243 if (!atomic_inc_not_zero(&priv
->chan
[ch
].submit_count
)) {
244 /* h/w fifo is full */
245 spin_unlock_irqrestore(&priv
->chan
[ch
].head_lock
, flags
);
249 head
= priv
->chan
[ch
].head
;
250 request
= &priv
->chan
[ch
].fifo
[head
];
252 /* map descriptor and save caller data */
254 desc
->hdr1
= desc
->hdr
;
256 request
->dma_desc
= dma_map_single(dev
, &desc
->hdr1
,
260 request
->dma_desc
= dma_map_single(dev
, desc
,
264 request
->callback
= callback
;
265 request
->context
= context
;
267 /* increment fifo head */
268 priv
->chan
[ch
].head
= (priv
->chan
[ch
].head
+ 1) & (priv
->fifo_len
- 1);
271 request
->desc
= desc
;
275 out_be32(priv
->chan
[ch
].reg
+ TALITOS_FF
,
276 upper_32_bits(request
->dma_desc
));
277 out_be32(priv
->chan
[ch
].reg
+ TALITOS_FF_LO
,
278 lower_32_bits(request
->dma_desc
));
280 spin_unlock_irqrestore(&priv
->chan
[ch
].head_lock
, flags
);
284 EXPORT_SYMBOL(talitos_submit
);
287 * process what was done, notify callback of error if not
289 static void flush_channel(struct device
*dev
, int ch
, int error
, int reset_ch
)
291 struct talitos_private
*priv
= dev_get_drvdata(dev
);
292 struct talitos_request
*request
, saved_req
;
295 bool is_sec1
= has_ftr_sec1(priv
);
297 spin_lock_irqsave(&priv
->chan
[ch
].tail_lock
, flags
);
299 tail
= priv
->chan
[ch
].tail
;
300 while (priv
->chan
[ch
].fifo
[tail
].desc
) {
303 request
= &priv
->chan
[ch
].fifo
[tail
];
305 /* descriptors with their done bits set don't get the error */
307 hdr
= is_sec1
? request
->desc
->hdr1
: request
->desc
->hdr
;
309 if ((hdr
& DESC_HDR_DONE
) == DESC_HDR_DONE
)
317 dma_unmap_single(dev
, request
->dma_desc
,
321 /* copy entries so we can call callback outside lock */
322 saved_req
.desc
= request
->desc
;
323 saved_req
.callback
= request
->callback
;
324 saved_req
.context
= request
->context
;
326 /* release request entry in fifo */
328 request
->desc
= NULL
;
330 /* increment fifo tail */
331 priv
->chan
[ch
].tail
= (tail
+ 1) & (priv
->fifo_len
- 1);
333 spin_unlock_irqrestore(&priv
->chan
[ch
].tail_lock
, flags
);
335 atomic_dec(&priv
->chan
[ch
].submit_count
);
337 saved_req
.callback(dev
, saved_req
.desc
, saved_req
.context
,
339 /* channel may resume processing in single desc error case */
340 if (error
&& !reset_ch
&& status
== error
)
342 spin_lock_irqsave(&priv
->chan
[ch
].tail_lock
, flags
);
343 tail
= priv
->chan
[ch
].tail
;
346 spin_unlock_irqrestore(&priv
->chan
[ch
].tail_lock
, flags
);
350 * process completed requests for channels that have done status
352 #define DEF_TALITOS_DONE(name, ch_done_mask) \
353 static void talitos_done_##name(unsigned long data) \
355 struct device *dev = (struct device *)data; \
356 struct talitos_private *priv = dev_get_drvdata(dev); \
357 unsigned long flags; \
359 if (ch_done_mask & 1) \
360 flush_channel(dev, 0, 0, 0); \
361 if (priv->num_channels == 1) \
363 if (ch_done_mask & (1 << 2)) \
364 flush_channel(dev, 1, 0, 0); \
365 if (ch_done_mask & (1 << 4)) \
366 flush_channel(dev, 2, 0, 0); \
367 if (ch_done_mask & (1 << 6)) \
368 flush_channel(dev, 3, 0, 0); \
371 /* At this point, all completed channels have been processed */ \
372 /* Unmask done interrupts for channels completed later on. */ \
373 spin_lock_irqsave(&priv->reg_lock, flags); \
374 setbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
375 setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT); \
376 spin_unlock_irqrestore(&priv->reg_lock, flags); \
378 DEF_TALITOS_DONE(4ch
, TALITOS_ISR_4CHDONE
)
379 DEF_TALITOS_DONE(ch0_2
, TALITOS_ISR_CH_0_2_DONE
)
380 DEF_TALITOS_DONE(ch1_3
, TALITOS_ISR_CH_1_3_DONE
)
383 * locate current (offending) descriptor
385 static u32
current_desc_hdr(struct device
*dev
, int ch
)
387 struct talitos_private
*priv
= dev_get_drvdata(dev
);
391 cur_desc
= ((u64
)in_be32(priv
->chan
[ch
].reg
+ TALITOS_CDPR
)) << 32;
392 cur_desc
|= in_be32(priv
->chan
[ch
].reg
+ TALITOS_CDPR_LO
);
395 dev_err(dev
, "CDPR is NULL, giving up search for offending descriptor\n");
399 tail
= priv
->chan
[ch
].tail
;
402 while (priv
->chan
[ch
].fifo
[iter
].dma_desc
!= cur_desc
) {
403 iter
= (iter
+ 1) & (priv
->fifo_len
- 1);
405 dev_err(dev
, "couldn't locate current descriptor\n");
410 return priv
->chan
[ch
].fifo
[iter
].desc
->hdr
;
414 * user diagnostics; report root cause of error based on execution unit status
416 static void report_eu_error(struct device
*dev
, int ch
, u32 desc_hdr
)
418 struct talitos_private
*priv
= dev_get_drvdata(dev
);
422 desc_hdr
= in_be32(priv
->chan
[ch
].reg
+ TALITOS_DESCBUF
);
424 switch (desc_hdr
& DESC_HDR_SEL0_MASK
) {
425 case DESC_HDR_SEL0_AFEU
:
426 dev_err(dev
, "AFEUISR 0x%08x_%08x\n",
427 in_be32(priv
->reg
+ TALITOS_AFEUISR
),
428 in_be32(priv
->reg
+ TALITOS_AFEUISR_LO
));
430 case DESC_HDR_SEL0_DEU
:
431 dev_err(dev
, "DEUISR 0x%08x_%08x\n",
432 in_be32(priv
->reg
+ TALITOS_DEUISR
),
433 in_be32(priv
->reg
+ TALITOS_DEUISR_LO
));
435 case DESC_HDR_SEL0_MDEUA
:
436 case DESC_HDR_SEL0_MDEUB
:
437 dev_err(dev
, "MDEUISR 0x%08x_%08x\n",
438 in_be32(priv
->reg
+ TALITOS_MDEUISR
),
439 in_be32(priv
->reg
+ TALITOS_MDEUISR_LO
));
441 case DESC_HDR_SEL0_RNG
:
442 dev_err(dev
, "RNGUISR 0x%08x_%08x\n",
443 in_be32(priv
->reg
+ TALITOS_RNGUISR
),
444 in_be32(priv
->reg
+ TALITOS_RNGUISR_LO
));
446 case DESC_HDR_SEL0_PKEU
:
447 dev_err(dev
, "PKEUISR 0x%08x_%08x\n",
448 in_be32(priv
->reg
+ TALITOS_PKEUISR
),
449 in_be32(priv
->reg
+ TALITOS_PKEUISR_LO
));
451 case DESC_HDR_SEL0_AESU
:
452 dev_err(dev
, "AESUISR 0x%08x_%08x\n",
453 in_be32(priv
->reg
+ TALITOS_AESUISR
),
454 in_be32(priv
->reg
+ TALITOS_AESUISR_LO
));
456 case DESC_HDR_SEL0_CRCU
:
457 dev_err(dev
, "CRCUISR 0x%08x_%08x\n",
458 in_be32(priv
->reg
+ TALITOS_CRCUISR
),
459 in_be32(priv
->reg
+ TALITOS_CRCUISR_LO
));
461 case DESC_HDR_SEL0_KEU
:
462 dev_err(dev
, "KEUISR 0x%08x_%08x\n",
463 in_be32(priv
->reg
+ TALITOS_KEUISR
),
464 in_be32(priv
->reg
+ TALITOS_KEUISR_LO
));
468 switch (desc_hdr
& DESC_HDR_SEL1_MASK
) {
469 case DESC_HDR_SEL1_MDEUA
:
470 case DESC_HDR_SEL1_MDEUB
:
471 dev_err(dev
, "MDEUISR 0x%08x_%08x\n",
472 in_be32(priv
->reg
+ TALITOS_MDEUISR
),
473 in_be32(priv
->reg
+ TALITOS_MDEUISR_LO
));
475 case DESC_HDR_SEL1_CRCU
:
476 dev_err(dev
, "CRCUISR 0x%08x_%08x\n",
477 in_be32(priv
->reg
+ TALITOS_CRCUISR
),
478 in_be32(priv
->reg
+ TALITOS_CRCUISR_LO
));
482 for (i
= 0; i
< 8; i
++)
483 dev_err(dev
, "DESCBUF 0x%08x_%08x\n",
484 in_be32(priv
->chan
[ch
].reg
+ TALITOS_DESCBUF
+ 8*i
),
485 in_be32(priv
->chan
[ch
].reg
+ TALITOS_DESCBUF_LO
+ 8*i
));
489 * recover from error interrupts
491 static void talitos_error(struct device
*dev
, u32 isr
, u32 isr_lo
)
493 struct talitos_private
*priv
= dev_get_drvdata(dev
);
494 unsigned int timeout
= TALITOS_TIMEOUT
;
495 int ch
, error
, reset_dev
= 0, reset_ch
= 0;
498 for (ch
= 0; ch
< priv
->num_channels
; ch
++) {
499 /* skip channels without errors */
500 if (!(isr
& (1 << (ch
* 2 + 1))))
505 v
= in_be32(priv
->chan
[ch
].reg
+ TALITOS_CCPSR
);
506 v_lo
= in_be32(priv
->chan
[ch
].reg
+ TALITOS_CCPSR_LO
);
508 if (v_lo
& TALITOS_CCPSR_LO_DOF
) {
509 dev_err(dev
, "double fetch fifo overflow error\n");
513 if (v_lo
& TALITOS_CCPSR_LO_SOF
) {
514 /* h/w dropped descriptor */
515 dev_err(dev
, "single fetch fifo overflow error\n");
518 if (v_lo
& TALITOS_CCPSR_LO_MDTE
)
519 dev_err(dev
, "master data transfer error\n");
520 if (v_lo
& TALITOS_CCPSR_LO_SGDLZ
)
521 dev_err(dev
, "s/g data length zero error\n");
522 if (v_lo
& TALITOS_CCPSR_LO_FPZ
)
523 dev_err(dev
, "fetch pointer zero error\n");
524 if (v_lo
& TALITOS_CCPSR_LO_IDH
)
525 dev_err(dev
, "illegal descriptor header error\n");
526 if (v_lo
& TALITOS_CCPSR_LO_IEU
)
527 dev_err(dev
, "invalid execution unit error\n");
528 if (v_lo
& TALITOS_CCPSR_LO_EU
)
529 report_eu_error(dev
, ch
, current_desc_hdr(dev
, ch
));
530 if (v_lo
& TALITOS_CCPSR_LO_GB
)
531 dev_err(dev
, "gather boundary error\n");
532 if (v_lo
& TALITOS_CCPSR_LO_GRL
)
533 dev_err(dev
, "gather return/length error\n");
534 if (v_lo
& TALITOS_CCPSR_LO_SB
)
535 dev_err(dev
, "scatter boundary error\n");
536 if (v_lo
& TALITOS_CCPSR_LO_SRL
)
537 dev_err(dev
, "scatter return/length error\n");
539 flush_channel(dev
, ch
, error
, reset_ch
);
542 reset_channel(dev
, ch
);
544 setbits32(priv
->chan
[ch
].reg
+ TALITOS_CCCR
,
546 setbits32(priv
->chan
[ch
].reg
+ TALITOS_CCCR_LO
, 0);
547 while ((in_be32(priv
->chan
[ch
].reg
+ TALITOS_CCCR
) &
548 TALITOS_CCCR_CONT
) && --timeout
)
551 dev_err(dev
, "failed to restart channel %d\n",
557 if (reset_dev
|| isr
& ~TALITOS_ISR_4CHERR
|| isr_lo
) {
558 dev_err(dev
, "done overflow, internal time out, or rngu error: "
559 "ISR 0x%08x_%08x\n", isr
, isr_lo
);
561 /* purge request queues */
562 for (ch
= 0; ch
< priv
->num_channels
; ch
++)
563 flush_channel(dev
, ch
, -EIO
, 1);
565 /* reset and reinitialize the device */
570 #define DEF_TALITOS_INTERRUPT(name, ch_done_mask, ch_err_mask, tlet) \
571 static irqreturn_t talitos_interrupt_##name(int irq, void *data) \
573 struct device *dev = data; \
574 struct talitos_private *priv = dev_get_drvdata(dev); \
576 unsigned long flags; \
578 spin_lock_irqsave(&priv->reg_lock, flags); \
579 isr = in_be32(priv->reg + TALITOS_ISR); \
580 isr_lo = in_be32(priv->reg + TALITOS_ISR_LO); \
581 /* Acknowledge interrupt */ \
582 out_be32(priv->reg + TALITOS_ICR, isr & (ch_done_mask | ch_err_mask)); \
583 out_be32(priv->reg + TALITOS_ICR_LO, isr_lo); \
585 if (unlikely(isr & ch_err_mask || isr_lo)) { \
586 spin_unlock_irqrestore(&priv->reg_lock, flags); \
587 talitos_error(dev, isr & ch_err_mask, isr_lo); \
590 if (likely(isr & ch_done_mask)) { \
591 /* mask further done interrupts. */ \
592 clrbits32(priv->reg + TALITOS_IMR, ch_done_mask); \
593 /* done_task will unmask done interrupts at exit */ \
594 tasklet_schedule(&priv->done_task[tlet]); \
596 spin_unlock_irqrestore(&priv->reg_lock, flags); \
599 return (isr & (ch_done_mask | ch_err_mask) || isr_lo) ? IRQ_HANDLED : \
602 DEF_TALITOS_INTERRUPT(4ch
, TALITOS_ISR_4CHDONE
, TALITOS_ISR_4CHERR
, 0)
603 DEF_TALITOS_INTERRUPT(ch0_2
, TALITOS_ISR_CH_0_2_DONE
, TALITOS_ISR_CH_0_2_ERR
, 0)
604 DEF_TALITOS_INTERRUPT(ch1_3
, TALITOS_ISR_CH_1_3_DONE
, TALITOS_ISR_CH_1_3_ERR
, 1)
609 static int talitos_rng_data_present(struct hwrng
*rng
, int wait
)
611 struct device
*dev
= (struct device
*)rng
->priv
;
612 struct talitos_private
*priv
= dev_get_drvdata(dev
);
616 for (i
= 0; i
< 20; i
++) {
617 ofl
= in_be32(priv
->reg
+ TALITOS_RNGUSR_LO
) &
618 TALITOS_RNGUSR_LO_OFL
;
627 static int talitos_rng_data_read(struct hwrng
*rng
, u32
*data
)
629 struct device
*dev
= (struct device
*)rng
->priv
;
630 struct talitos_private
*priv
= dev_get_drvdata(dev
);
632 /* rng fifo requires 64-bit accesses */
633 *data
= in_be32(priv
->reg
+ TALITOS_RNGU_FIFO
);
634 *data
= in_be32(priv
->reg
+ TALITOS_RNGU_FIFO_LO
);
639 static int talitos_rng_init(struct hwrng
*rng
)
641 struct device
*dev
= (struct device
*)rng
->priv
;
642 struct talitos_private
*priv
= dev_get_drvdata(dev
);
643 unsigned int timeout
= TALITOS_TIMEOUT
;
645 setbits32(priv
->reg
+ TALITOS_RNGURCR_LO
, TALITOS_RNGURCR_LO_SR
);
646 while (!(in_be32(priv
->reg
+ TALITOS_RNGUSR_LO
) & TALITOS_RNGUSR_LO_RD
)
650 dev_err(dev
, "failed to reset rng hw\n");
654 /* start generating */
655 setbits32(priv
->reg
+ TALITOS_RNGUDSR_LO
, 0);
660 static int talitos_register_rng(struct device
*dev
)
662 struct talitos_private
*priv
= dev_get_drvdata(dev
);
664 priv
->rng
.name
= dev_driver_string(dev
),
665 priv
->rng
.init
= talitos_rng_init
,
666 priv
->rng
.data_present
= talitos_rng_data_present
,
667 priv
->rng
.data_read
= talitos_rng_data_read
,
668 priv
->rng
.priv
= (unsigned long)dev
;
670 return hwrng_register(&priv
->rng
);
673 static void talitos_unregister_rng(struct device
*dev
)
675 struct talitos_private
*priv
= dev_get_drvdata(dev
);
677 hwrng_unregister(&priv
->rng
);
683 #define TALITOS_CRA_PRIORITY 3000
684 #define TALITOS_MAX_KEY_SIZE 96
685 #define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
690 __be32 desc_hdr_template
;
691 u8 key
[TALITOS_MAX_KEY_SIZE
];
692 u8 iv
[TALITOS_MAX_IV_LENGTH
];
694 unsigned int enckeylen
;
695 unsigned int authkeylen
;
696 unsigned int authsize
;
699 #define HASH_MAX_BLOCK_SIZE SHA512_BLOCK_SIZE
700 #define TALITOS_MDEU_MAX_CONTEXT_SIZE TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
702 struct talitos_ahash_req_ctx
{
703 u32 hw_context
[TALITOS_MDEU_MAX_CONTEXT_SIZE
/ sizeof(u32
)];
704 unsigned int hw_context_size
;
705 u8 buf
[HASH_MAX_BLOCK_SIZE
];
706 u8 bufnext
[HASH_MAX_BLOCK_SIZE
];
710 unsigned int to_hash_later
;
712 struct scatterlist bufsl
[2];
713 struct scatterlist
*psrc
;
716 static int aead_setauthsize(struct crypto_aead
*authenc
,
717 unsigned int authsize
)
719 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
721 ctx
->authsize
= authsize
;
726 static int aead_setkey(struct crypto_aead
*authenc
,
727 const u8
*key
, unsigned int keylen
)
729 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
730 struct crypto_authenc_keys keys
;
732 if (crypto_authenc_extractkeys(&keys
, key
, keylen
) != 0)
735 if (keys
.authkeylen
+ keys
.enckeylen
> TALITOS_MAX_KEY_SIZE
)
738 memcpy(ctx
->key
, keys
.authkey
, keys
.authkeylen
);
739 memcpy(&ctx
->key
[keys
.authkeylen
], keys
.enckey
, keys
.enckeylen
);
741 ctx
->keylen
= keys
.authkeylen
+ keys
.enckeylen
;
742 ctx
->enckeylen
= keys
.enckeylen
;
743 ctx
->authkeylen
= keys
.authkeylen
;
748 crypto_aead_set_flags(authenc
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
753 * talitos_edesc - s/w-extended descriptor
754 * @assoc_nents: number of segments in associated data scatterlist
755 * @src_nents: number of segments in input scatterlist
756 * @dst_nents: number of segments in output scatterlist
757 * @assoc_chained: whether assoc is chained or not
758 * @src_chained: whether src is chained or not
759 * @dst_chained: whether dst is chained or not
760 * @iv_dma: dma address of iv for checking continuity and link table
761 * @dma_len: length of dma mapped link_tbl space
762 * @dma_link_tbl: bus physical address of link_tbl
763 * @desc: h/w descriptor
764 * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1)
766 * if decrypting (with authcheck), or either one of src_nents or dst_nents
767 * is greater than 1, an integrity check value is concatenated to the end
770 struct talitos_edesc
{
779 dma_addr_t dma_link_tbl
;
780 struct talitos_desc desc
;
781 struct talitos_ptr link_tbl
[0];
784 static int talitos_map_sg(struct device
*dev
, struct scatterlist
*sg
,
785 unsigned int nents
, enum dma_data_direction dir
,
788 if (unlikely(chained
))
790 dma_map_sg(dev
, sg
, 1, dir
);
794 dma_map_sg(dev
, sg
, nents
, dir
);
798 static void talitos_unmap_sg_chain(struct device
*dev
, struct scatterlist
*sg
,
799 enum dma_data_direction dir
)
802 dma_unmap_sg(dev
, sg
, 1, dir
);
807 static void talitos_sg_unmap(struct device
*dev
,
808 struct talitos_edesc
*edesc
,
809 struct scatterlist
*src
,
810 struct scatterlist
*dst
)
812 unsigned int src_nents
= edesc
->src_nents
? : 1;
813 unsigned int dst_nents
= edesc
->dst_nents
? : 1;
816 if (edesc
->src_chained
)
817 talitos_unmap_sg_chain(dev
, src
, DMA_TO_DEVICE
);
819 dma_unmap_sg(dev
, src
, src_nents
, DMA_TO_DEVICE
);
822 if (edesc
->dst_chained
)
823 talitos_unmap_sg_chain(dev
, dst
,
826 dma_unmap_sg(dev
, dst
, dst_nents
,
830 if (edesc
->src_chained
)
831 talitos_unmap_sg_chain(dev
, src
, DMA_BIDIRECTIONAL
);
833 dma_unmap_sg(dev
, src
, src_nents
, DMA_BIDIRECTIONAL
);
836 static void ipsec_esp_unmap(struct device
*dev
,
837 struct talitos_edesc
*edesc
,
838 struct aead_request
*areq
)
840 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[6], DMA_FROM_DEVICE
);
841 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[3], DMA_TO_DEVICE
);
842 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[2], DMA_TO_DEVICE
);
843 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[0], DMA_TO_DEVICE
);
845 if (edesc
->assoc_chained
)
846 talitos_unmap_sg_chain(dev
, areq
->assoc
, DMA_TO_DEVICE
);
847 else if (areq
->assoclen
)
848 /* assoc_nents counts also for IV in non-contiguous cases */
849 dma_unmap_sg(dev
, areq
->assoc
,
850 edesc
->assoc_nents
? edesc
->assoc_nents
- 1 : 1,
853 talitos_sg_unmap(dev
, edesc
, areq
->src
, areq
->dst
);
856 dma_unmap_single(dev
, edesc
->dma_link_tbl
, edesc
->dma_len
,
861 * ipsec_esp descriptor callbacks
863 static void ipsec_esp_encrypt_done(struct device
*dev
,
864 struct talitos_desc
*desc
, void *context
,
867 struct aead_request
*areq
= context
;
868 struct crypto_aead
*authenc
= crypto_aead_reqtfm(areq
);
869 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
870 struct talitos_edesc
*edesc
;
871 struct scatterlist
*sg
;
874 edesc
= container_of(desc
, struct talitos_edesc
, desc
);
876 ipsec_esp_unmap(dev
, edesc
, areq
);
878 /* copy the generated ICV to dst */
879 if (edesc
->dst_nents
) {
880 icvdata
= &edesc
->link_tbl
[edesc
->src_nents
+
881 edesc
->dst_nents
+ 2 +
883 sg
= sg_last(areq
->dst
, edesc
->dst_nents
);
884 memcpy((char *)sg_virt(sg
) + sg
->length
- ctx
->authsize
,
885 icvdata
, ctx
->authsize
);
890 aead_request_complete(areq
, err
);
893 static void ipsec_esp_decrypt_swauth_done(struct device
*dev
,
894 struct talitos_desc
*desc
,
895 void *context
, int err
)
897 struct aead_request
*req
= context
;
898 struct crypto_aead
*authenc
= crypto_aead_reqtfm(req
);
899 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
900 struct talitos_edesc
*edesc
;
901 struct scatterlist
*sg
;
904 edesc
= container_of(desc
, struct talitos_edesc
, desc
);
906 ipsec_esp_unmap(dev
, edesc
, req
);
911 icvdata
= &edesc
->link_tbl
[edesc
->src_nents
+
912 edesc
->dst_nents
+ 2 +
915 icvdata
= &edesc
->link_tbl
[0];
917 sg
= sg_last(req
->dst
, edesc
->dst_nents
? : 1);
918 err
= memcmp(icvdata
, (char *)sg_virt(sg
) + sg
->length
-
919 ctx
->authsize
, ctx
->authsize
) ? -EBADMSG
: 0;
924 aead_request_complete(req
, err
);
927 static void ipsec_esp_decrypt_hwauth_done(struct device
*dev
,
928 struct talitos_desc
*desc
,
929 void *context
, int err
)
931 struct aead_request
*req
= context
;
932 struct talitos_edesc
*edesc
;
934 edesc
= container_of(desc
, struct talitos_edesc
, desc
);
936 ipsec_esp_unmap(dev
, edesc
, req
);
938 /* check ICV auth status */
939 if (!err
&& ((desc
->hdr_lo
& DESC_HDR_LO_ICCR1_MASK
) !=
940 DESC_HDR_LO_ICCR1_PASS
))
945 aead_request_complete(req
, err
);
949 * convert scatterlist to SEC h/w link table format
950 * stop at cryptlen bytes
952 static int sg_to_link_tbl(struct scatterlist
*sg
, int sg_count
,
953 int cryptlen
, struct talitos_ptr
*link_tbl_ptr
)
958 to_talitos_ptr(link_tbl_ptr
, sg_dma_address(sg
), 0);
959 link_tbl_ptr
->len
= cpu_to_be16(sg_dma_len(sg
));
960 link_tbl_ptr
->j_extent
= 0;
962 cryptlen
-= sg_dma_len(sg
);
966 /* adjust (decrease) last one (or two) entry's len to cryptlen */
968 while (be16_to_cpu(link_tbl_ptr
->len
) <= (-cryptlen
)) {
969 /* Empty this entry, and move to previous one */
970 cryptlen
+= be16_to_cpu(link_tbl_ptr
->len
);
971 link_tbl_ptr
->len
= 0;
975 be16_add_cpu(&link_tbl_ptr
->len
, cryptlen
);
977 /* tag end of link table */
978 link_tbl_ptr
->j_extent
= DESC_PTR_LNKTBL_RETURN
;
984 * fill in and submit ipsec_esp descriptor
986 static int ipsec_esp(struct talitos_edesc
*edesc
, struct aead_request
*areq
,
987 u64 seq
, void (*callback
) (struct device
*dev
,
988 struct talitos_desc
*desc
,
989 void *context
, int error
))
991 struct crypto_aead
*aead
= crypto_aead_reqtfm(areq
);
992 struct talitos_ctx
*ctx
= crypto_aead_ctx(aead
);
993 struct device
*dev
= ctx
->dev
;
994 struct talitos_desc
*desc
= &edesc
->desc
;
995 unsigned int cryptlen
= areq
->cryptlen
;
996 unsigned int authsize
= ctx
->authsize
;
997 unsigned int ivsize
= crypto_aead_ivsize(aead
);
1002 map_single_talitos_ptr(dev
, &desc
->ptr
[0], ctx
->authkeylen
, &ctx
->key
,
1006 desc
->ptr
[1].len
= cpu_to_be16(areq
->assoclen
+ ivsize
);
1007 if (edesc
->assoc_nents
) {
1008 int tbl_off
= edesc
->src_nents
+ edesc
->dst_nents
+ 2;
1009 struct talitos_ptr
*tbl_ptr
= &edesc
->link_tbl
[tbl_off
];
1011 to_talitos_ptr(&desc
->ptr
[1], edesc
->dma_link_tbl
+ tbl_off
*
1012 sizeof(struct talitos_ptr
), 0);
1013 desc
->ptr
[1].j_extent
= DESC_PTR_LNKTBL_JUMP
;
1015 /* assoc_nents - 1 entries for assoc, 1 for IV */
1016 sg_count
= sg_to_link_tbl(areq
->assoc
, edesc
->assoc_nents
- 1,
1017 areq
->assoclen
, tbl_ptr
);
1019 /* add IV to link table */
1020 tbl_ptr
+= sg_count
- 1;
1021 tbl_ptr
->j_extent
= 0;
1023 to_talitos_ptr(tbl_ptr
, edesc
->iv_dma
, 0);
1024 tbl_ptr
->len
= cpu_to_be16(ivsize
);
1025 tbl_ptr
->j_extent
= DESC_PTR_LNKTBL_RETURN
;
1027 dma_sync_single_for_device(dev
, edesc
->dma_link_tbl
,
1028 edesc
->dma_len
, DMA_BIDIRECTIONAL
);
1031 to_talitos_ptr(&desc
->ptr
[1],
1032 sg_dma_address(areq
->assoc
), 0);
1034 to_talitos_ptr(&desc
->ptr
[1], edesc
->iv_dma
, 0);
1035 desc
->ptr
[1].j_extent
= 0;
1039 to_talitos_ptr(&desc
->ptr
[2], edesc
->iv_dma
, 0);
1040 desc
->ptr
[2].len
= cpu_to_be16(ivsize
);
1041 desc
->ptr
[2].j_extent
= 0;
1042 /* Sync needed for the aead_givencrypt case */
1043 dma_sync_single_for_device(dev
, edesc
->iv_dma
, ivsize
, DMA_TO_DEVICE
);
1046 map_single_talitos_ptr(dev
, &desc
->ptr
[3], ctx
->enckeylen
,
1047 (char *)&ctx
->key
+ ctx
->authkeylen
,
1052 * map and adjust cipher len to aead request cryptlen.
1053 * extent is bytes of HMAC postpended to ciphertext,
1054 * typically 12 for ipsec
1056 desc
->ptr
[4].len
= cpu_to_be16(cryptlen
);
1057 desc
->ptr
[4].j_extent
= authsize
;
1059 sg_count
= talitos_map_sg(dev
, areq
->src
, edesc
->src_nents
? : 1,
1060 (areq
->src
== areq
->dst
) ? DMA_BIDIRECTIONAL
1062 edesc
->src_chained
);
1064 if (sg_count
== 1) {
1065 to_talitos_ptr(&desc
->ptr
[4], sg_dma_address(areq
->src
), 0);
1067 sg_link_tbl_len
= cryptlen
;
1069 if (edesc
->desc
.hdr
& DESC_HDR_MODE1_MDEU_CICV
)
1070 sg_link_tbl_len
= cryptlen
+ authsize
;
1072 sg_count
= sg_to_link_tbl(areq
->src
, sg_count
, sg_link_tbl_len
,
1073 &edesc
->link_tbl
[0]);
1075 desc
->ptr
[4].j_extent
|= DESC_PTR_LNKTBL_JUMP
;
1076 to_talitos_ptr(&desc
->ptr
[4], edesc
->dma_link_tbl
, 0);
1077 dma_sync_single_for_device(dev
, edesc
->dma_link_tbl
,
1081 /* Only one segment now, so no link tbl needed */
1082 to_talitos_ptr(&desc
->ptr
[4],
1083 sg_dma_address(areq
->src
), 0);
1088 desc
->ptr
[5].len
= cpu_to_be16(cryptlen
);
1089 desc
->ptr
[5].j_extent
= authsize
;
1091 if (areq
->src
!= areq
->dst
)
1092 sg_count
= talitos_map_sg(dev
, areq
->dst
,
1093 edesc
->dst_nents
? : 1,
1094 DMA_FROM_DEVICE
, edesc
->dst_chained
);
1096 if (sg_count
== 1) {
1097 to_talitos_ptr(&desc
->ptr
[5], sg_dma_address(areq
->dst
), 0);
1099 int tbl_off
= edesc
->src_nents
+ 1;
1100 struct talitos_ptr
*tbl_ptr
= &edesc
->link_tbl
[tbl_off
];
1102 to_talitos_ptr(&desc
->ptr
[5], edesc
->dma_link_tbl
+
1103 tbl_off
* sizeof(struct talitos_ptr
), 0);
1104 sg_count
= sg_to_link_tbl(areq
->dst
, sg_count
, cryptlen
,
1107 /* Add an entry to the link table for ICV data */
1108 tbl_ptr
+= sg_count
- 1;
1109 tbl_ptr
->j_extent
= 0;
1111 tbl_ptr
->j_extent
= DESC_PTR_LNKTBL_RETURN
;
1112 tbl_ptr
->len
= cpu_to_be16(authsize
);
1114 /* icv data follows link tables */
1115 to_talitos_ptr(tbl_ptr
, edesc
->dma_link_tbl
+
1116 (tbl_off
+ edesc
->dst_nents
+ 1 +
1117 edesc
->assoc_nents
) *
1118 sizeof(struct talitos_ptr
), 0);
1119 desc
->ptr
[5].j_extent
|= DESC_PTR_LNKTBL_JUMP
;
1120 dma_sync_single_for_device(ctx
->dev
, edesc
->dma_link_tbl
,
1121 edesc
->dma_len
, DMA_BIDIRECTIONAL
);
1125 map_single_talitos_ptr(dev
, &desc
->ptr
[6], ivsize
, ctx
->iv
,
1128 ret
= talitos_submit(dev
, ctx
->ch
, desc
, callback
, areq
);
1129 if (ret
!= -EINPROGRESS
) {
1130 ipsec_esp_unmap(dev
, edesc
, areq
);
1137 * derive number of elements in scatterlist
1139 static int sg_count(struct scatterlist
*sg_list
, int nbytes
, bool *chained
)
1141 struct scatterlist
*sg
= sg_list
;
1145 while (nbytes
> 0) {
1147 nbytes
-= sg
->length
;
1148 if (!sg_is_last(sg
) && (sg
+ 1)->length
== 0)
1157 * allocate and map the extended descriptor
1159 static struct talitos_edesc
*talitos_edesc_alloc(struct device
*dev
,
1160 struct scatterlist
*assoc
,
1161 struct scatterlist
*src
,
1162 struct scatterlist
*dst
,
1164 unsigned int assoclen
,
1165 unsigned int cryptlen
,
1166 unsigned int authsize
,
1167 unsigned int ivsize
,
1172 struct talitos_edesc
*edesc
;
1173 int assoc_nents
= 0, src_nents
, dst_nents
, alloc_len
, dma_len
;
1174 bool assoc_chained
= false, src_chained
= false, dst_chained
= false;
1175 dma_addr_t iv_dma
= 0;
1176 gfp_t flags
= cryptoflags
& CRYPTO_TFM_REQ_MAY_SLEEP
? GFP_KERNEL
:
1179 if (cryptlen
+ authsize
> TALITOS_MAX_DATA_LEN
) {
1180 dev_err(dev
, "length exceeds h/w max limit\n");
1181 return ERR_PTR(-EINVAL
);
1185 iv_dma
= dma_map_single(dev
, iv
, ivsize
, DMA_TO_DEVICE
);
1189 * Currently it is assumed that iv is provided whenever assoc
1194 assoc_nents
= sg_count(assoc
, assoclen
, &assoc_chained
);
1195 talitos_map_sg(dev
, assoc
, assoc_nents
, DMA_TO_DEVICE
,
1197 assoc_nents
= (assoc_nents
== 1) ? 0 : assoc_nents
;
1199 if (assoc_nents
|| sg_dma_address(assoc
) + assoclen
!= iv_dma
)
1200 assoc_nents
= assoc_nents
? assoc_nents
+ 1 : 2;
1203 if (!dst
|| dst
== src
) {
1204 src_nents
= sg_count(src
, cryptlen
+ authsize
, &src_chained
);
1205 src_nents
= (src_nents
== 1) ? 0 : src_nents
;
1206 dst_nents
= dst
? src_nents
: 0;
1207 } else { /* dst && dst != src*/
1208 src_nents
= sg_count(src
, cryptlen
+ (encrypt
? 0 : authsize
),
1210 src_nents
= (src_nents
== 1) ? 0 : src_nents
;
1211 dst_nents
= sg_count(dst
, cryptlen
+ (encrypt
? authsize
: 0),
1213 dst_nents
= (dst_nents
== 1) ? 0 : dst_nents
;
1217 * allocate space for base edesc plus the link tables,
1218 * allowing for two separate entries for ICV and generated ICV (+ 2),
1219 * and the ICV data itself
1221 alloc_len
= sizeof(struct talitos_edesc
);
1222 if (assoc_nents
|| src_nents
|| dst_nents
) {
1223 dma_len
= (src_nents
+ dst_nents
+ 2 + assoc_nents
) *
1224 sizeof(struct talitos_ptr
) + authsize
;
1225 alloc_len
+= dma_len
;
1228 alloc_len
+= icv_stashing
? authsize
: 0;
1231 edesc
= kmalloc(alloc_len
, GFP_DMA
| flags
);
1234 talitos_unmap_sg_chain(dev
, assoc
, DMA_TO_DEVICE
);
1236 dma_unmap_sg(dev
, assoc
,
1237 assoc_nents
? assoc_nents
- 1 : 1,
1241 dma_unmap_single(dev
, iv_dma
, ivsize
, DMA_TO_DEVICE
);
1243 dev_err(dev
, "could not allocate edescriptor\n");
1244 return ERR_PTR(-ENOMEM
);
1247 edesc
->assoc_nents
= assoc_nents
;
1248 edesc
->src_nents
= src_nents
;
1249 edesc
->dst_nents
= dst_nents
;
1250 edesc
->assoc_chained
= assoc_chained
;
1251 edesc
->src_chained
= src_chained
;
1252 edesc
->dst_chained
= dst_chained
;
1253 edesc
->iv_dma
= iv_dma
;
1254 edesc
->dma_len
= dma_len
;
1256 edesc
->dma_link_tbl
= dma_map_single(dev
, &edesc
->link_tbl
[0],
1263 static struct talitos_edesc
*aead_edesc_alloc(struct aead_request
*areq
, u8
*iv
,
1264 int icv_stashing
, bool encrypt
)
1266 struct crypto_aead
*authenc
= crypto_aead_reqtfm(areq
);
1267 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
1268 unsigned int ivsize
= crypto_aead_ivsize(authenc
);
1270 return talitos_edesc_alloc(ctx
->dev
, areq
->assoc
, areq
->src
, areq
->dst
,
1271 iv
, areq
->assoclen
, areq
->cryptlen
,
1272 ctx
->authsize
, ivsize
, icv_stashing
,
1273 areq
->base
.flags
, encrypt
);
1276 static int aead_encrypt(struct aead_request
*req
)
1278 struct crypto_aead
*authenc
= crypto_aead_reqtfm(req
);
1279 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
1280 struct talitos_edesc
*edesc
;
1282 /* allocate extended descriptor */
1283 edesc
= aead_edesc_alloc(req
, req
->iv
, 0, true);
1285 return PTR_ERR(edesc
);
1288 edesc
->desc
.hdr
= ctx
->desc_hdr_template
| DESC_HDR_MODE0_ENCRYPT
;
1290 return ipsec_esp(edesc
, req
, 0, ipsec_esp_encrypt_done
);
1293 static int aead_decrypt(struct aead_request
*req
)
1295 struct crypto_aead
*authenc
= crypto_aead_reqtfm(req
);
1296 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
1297 unsigned int authsize
= ctx
->authsize
;
1298 struct talitos_private
*priv
= dev_get_drvdata(ctx
->dev
);
1299 struct talitos_edesc
*edesc
;
1300 struct scatterlist
*sg
;
1303 req
->cryptlen
-= authsize
;
1305 /* allocate extended descriptor */
1306 edesc
= aead_edesc_alloc(req
, req
->iv
, 1, false);
1308 return PTR_ERR(edesc
);
1310 if ((priv
->features
& TALITOS_FTR_HW_AUTH_CHECK
) &&
1311 ((!edesc
->src_nents
&& !edesc
->dst_nents
) ||
1312 priv
->features
& TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT
)) {
1314 /* decrypt and check the ICV */
1315 edesc
->desc
.hdr
= ctx
->desc_hdr_template
|
1316 DESC_HDR_DIR_INBOUND
|
1317 DESC_HDR_MODE1_MDEU_CICV
;
1319 /* reset integrity check result bits */
1320 edesc
->desc
.hdr_lo
= 0;
1322 return ipsec_esp(edesc
, req
, 0, ipsec_esp_decrypt_hwauth_done
);
1325 /* Have to check the ICV with software */
1326 edesc
->desc
.hdr
= ctx
->desc_hdr_template
| DESC_HDR_DIR_INBOUND
;
1328 /* stash incoming ICV for later cmp with ICV generated by the h/w */
1330 icvdata
= &edesc
->link_tbl
[edesc
->src_nents
+
1331 edesc
->dst_nents
+ 2 +
1332 edesc
->assoc_nents
];
1334 icvdata
= &edesc
->link_tbl
[0];
1336 sg
= sg_last(req
->src
, edesc
->src_nents
? : 1);
1338 memcpy(icvdata
, (char *)sg_virt(sg
) + sg
->length
- ctx
->authsize
,
1341 return ipsec_esp(edesc
, req
, 0, ipsec_esp_decrypt_swauth_done
);
1344 static int aead_givencrypt(struct aead_givcrypt_request
*req
)
1346 struct aead_request
*areq
= &req
->areq
;
1347 struct crypto_aead
*authenc
= crypto_aead_reqtfm(areq
);
1348 struct talitos_ctx
*ctx
= crypto_aead_ctx(authenc
);
1349 struct talitos_edesc
*edesc
;
1351 /* allocate extended descriptor */
1352 edesc
= aead_edesc_alloc(areq
, req
->giv
, 0, true);
1354 return PTR_ERR(edesc
);
1357 edesc
->desc
.hdr
= ctx
->desc_hdr_template
| DESC_HDR_MODE0_ENCRYPT
;
1359 memcpy(req
->giv
, ctx
->iv
, crypto_aead_ivsize(authenc
));
1360 /* avoid consecutive packets going out with same IV */
1361 *(__be64
*)req
->giv
^= cpu_to_be64(req
->seq
);
1363 return ipsec_esp(edesc
, areq
, req
->seq
, ipsec_esp_encrypt_done
);
1366 static int ablkcipher_setkey(struct crypto_ablkcipher
*cipher
,
1367 const u8
*key
, unsigned int keylen
)
1369 struct talitos_ctx
*ctx
= crypto_ablkcipher_ctx(cipher
);
1371 memcpy(&ctx
->key
, key
, keylen
);
1372 ctx
->keylen
= keylen
;
1377 static void unmap_sg_talitos_ptr(struct device
*dev
, struct scatterlist
*src
,
1378 struct scatterlist
*dst
, unsigned int len
,
1379 struct talitos_edesc
*edesc
)
1381 talitos_sg_unmap(dev
, edesc
, src
, dst
);
1384 static void common_nonsnoop_unmap(struct device
*dev
,
1385 struct talitos_edesc
*edesc
,
1386 struct ablkcipher_request
*areq
)
1388 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[5], DMA_FROM_DEVICE
);
1390 unmap_sg_talitos_ptr(dev
, areq
->src
, areq
->dst
, areq
->nbytes
, edesc
);
1391 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[2], DMA_TO_DEVICE
);
1392 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[1], DMA_TO_DEVICE
);
1395 dma_unmap_single(dev
, edesc
->dma_link_tbl
, edesc
->dma_len
,
1399 static void ablkcipher_done(struct device
*dev
,
1400 struct talitos_desc
*desc
, void *context
,
1403 struct ablkcipher_request
*areq
= context
;
1404 struct talitos_edesc
*edesc
;
1406 edesc
= container_of(desc
, struct talitos_edesc
, desc
);
1408 common_nonsnoop_unmap(dev
, edesc
, areq
);
1412 areq
->base
.complete(&areq
->base
, err
);
1415 int map_sg_in_talitos_ptr(struct device
*dev
, struct scatterlist
*src
,
1416 unsigned int len
, struct talitos_edesc
*edesc
,
1417 enum dma_data_direction dir
, struct talitos_ptr
*ptr
)
1420 struct talitos_private
*priv
= dev_get_drvdata(dev
);
1421 bool is_sec1
= has_ftr_sec1(priv
);
1423 to_talitos_ptr_len(ptr
, len
, is_sec1
);
1424 to_talitos_ptr_extent_clear(ptr
, is_sec1
);
1426 sg_count
= talitos_map_sg(dev
, src
, edesc
->src_nents
? : 1, dir
,
1427 edesc
->src_chained
);
1429 if (sg_count
== 1) {
1430 to_talitos_ptr(ptr
, sg_dma_address(src
), is_sec1
);
1432 sg_count
= sg_to_link_tbl(src
, sg_count
, len
,
1433 &edesc
->link_tbl
[0]);
1435 to_talitos_ptr(ptr
, edesc
->dma_link_tbl
, 0);
1436 ptr
->j_extent
|= DESC_PTR_LNKTBL_JUMP
;
1437 dma_sync_single_for_device(dev
, edesc
->dma_link_tbl
,
1441 /* Only one segment now, so no link tbl needed */
1442 to_talitos_ptr(ptr
, sg_dma_address(src
), is_sec1
);
1448 void map_sg_out_talitos_ptr(struct device
*dev
, struct scatterlist
*dst
,
1449 unsigned int len
, struct talitos_edesc
*edesc
,
1450 enum dma_data_direction dir
,
1451 struct talitos_ptr
*ptr
, int sg_count
)
1453 struct talitos_private
*priv
= dev_get_drvdata(dev
);
1454 bool is_sec1
= has_ftr_sec1(priv
);
1456 to_talitos_ptr_len(ptr
, len
, is_sec1
);
1457 to_talitos_ptr_extent_clear(ptr
, is_sec1
);
1459 if (dir
!= DMA_NONE
)
1460 sg_count
= talitos_map_sg(dev
, dst
, edesc
->dst_nents
? : 1,
1461 dir
, edesc
->dst_chained
);
1463 if (sg_count
== 1) {
1464 to_talitos_ptr(ptr
, sg_dma_address(dst
), is_sec1
);
1466 struct talitos_ptr
*link_tbl_ptr
=
1467 &edesc
->link_tbl
[edesc
->src_nents
+ 1];
1469 to_talitos_ptr(ptr
, edesc
->dma_link_tbl
+
1470 (edesc
->src_nents
+ 1) *
1471 sizeof(struct talitos_ptr
), 0);
1472 ptr
->j_extent
|= DESC_PTR_LNKTBL_JUMP
;
1473 sg_count
= sg_to_link_tbl(dst
, sg_count
, len
, link_tbl_ptr
);
1474 dma_sync_single_for_device(dev
, edesc
->dma_link_tbl
,
1475 edesc
->dma_len
, DMA_BIDIRECTIONAL
);
1479 static int common_nonsnoop(struct talitos_edesc
*edesc
,
1480 struct ablkcipher_request
*areq
,
1481 void (*callback
) (struct device
*dev
,
1482 struct talitos_desc
*desc
,
1483 void *context
, int error
))
1485 struct crypto_ablkcipher
*cipher
= crypto_ablkcipher_reqtfm(areq
);
1486 struct talitos_ctx
*ctx
= crypto_ablkcipher_ctx(cipher
);
1487 struct device
*dev
= ctx
->dev
;
1488 struct talitos_desc
*desc
= &edesc
->desc
;
1489 unsigned int cryptlen
= areq
->nbytes
;
1490 unsigned int ivsize
= crypto_ablkcipher_ivsize(cipher
);
1492 struct talitos_private
*priv
= dev_get_drvdata(dev
);
1493 bool is_sec1
= has_ftr_sec1(priv
);
1495 /* first DWORD empty */
1496 desc
->ptr
[0] = zero_entry
;
1499 to_talitos_ptr(&desc
->ptr
[1], edesc
->iv_dma
, is_sec1
);
1500 to_talitos_ptr_len(&desc
->ptr
[1], ivsize
, is_sec1
);
1501 to_talitos_ptr_extent_clear(&desc
->ptr
[1], is_sec1
);
1504 map_single_talitos_ptr(dev
, &desc
->ptr
[2], ctx
->keylen
,
1505 (char *)&ctx
->key
, DMA_TO_DEVICE
);
1510 sg_count
= map_sg_in_talitos_ptr(dev
, areq
->src
, cryptlen
, edesc
,
1511 (areq
->src
== areq
->dst
) ?
1512 DMA_BIDIRECTIONAL
: DMA_TO_DEVICE
,
1516 map_sg_out_talitos_ptr(dev
, areq
->dst
, cryptlen
, edesc
,
1517 (areq
->src
== areq
->dst
) ? DMA_NONE
1519 &desc
->ptr
[4], sg_count
);
1522 map_single_talitos_ptr(dev
, &desc
->ptr
[5], ivsize
, ctx
->iv
,
1525 /* last DWORD empty */
1526 desc
->ptr
[6] = zero_entry
;
1528 ret
= talitos_submit(dev
, ctx
->ch
, desc
, callback
, areq
);
1529 if (ret
!= -EINPROGRESS
) {
1530 common_nonsnoop_unmap(dev
, edesc
, areq
);
1536 static struct talitos_edesc
*ablkcipher_edesc_alloc(struct ablkcipher_request
*
1539 struct crypto_ablkcipher
*cipher
= crypto_ablkcipher_reqtfm(areq
);
1540 struct talitos_ctx
*ctx
= crypto_ablkcipher_ctx(cipher
);
1541 unsigned int ivsize
= crypto_ablkcipher_ivsize(cipher
);
1543 return talitos_edesc_alloc(ctx
->dev
, NULL
, areq
->src
, areq
->dst
,
1544 areq
->info
, 0, areq
->nbytes
, 0, ivsize
, 0,
1545 areq
->base
.flags
, encrypt
);
1548 static int ablkcipher_encrypt(struct ablkcipher_request
*areq
)
1550 struct crypto_ablkcipher
*cipher
= crypto_ablkcipher_reqtfm(areq
);
1551 struct talitos_ctx
*ctx
= crypto_ablkcipher_ctx(cipher
);
1552 struct talitos_edesc
*edesc
;
1554 /* allocate extended descriptor */
1555 edesc
= ablkcipher_edesc_alloc(areq
, true);
1557 return PTR_ERR(edesc
);
1560 edesc
->desc
.hdr
= ctx
->desc_hdr_template
| DESC_HDR_MODE0_ENCRYPT
;
1562 return common_nonsnoop(edesc
, areq
, ablkcipher_done
);
1565 static int ablkcipher_decrypt(struct ablkcipher_request
*areq
)
1567 struct crypto_ablkcipher
*cipher
= crypto_ablkcipher_reqtfm(areq
);
1568 struct talitos_ctx
*ctx
= crypto_ablkcipher_ctx(cipher
);
1569 struct talitos_edesc
*edesc
;
1571 /* allocate extended descriptor */
1572 edesc
= ablkcipher_edesc_alloc(areq
, false);
1574 return PTR_ERR(edesc
);
1576 edesc
->desc
.hdr
= ctx
->desc_hdr_template
| DESC_HDR_DIR_INBOUND
;
1578 return common_nonsnoop(edesc
, areq
, ablkcipher_done
);
1581 static void common_nonsnoop_hash_unmap(struct device
*dev
,
1582 struct talitos_edesc
*edesc
,
1583 struct ahash_request
*areq
)
1585 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1586 struct talitos_private
*priv
= dev_get_drvdata(dev
);
1587 bool is_sec1
= has_ftr_sec1(priv
);
1589 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[5], DMA_FROM_DEVICE
);
1591 unmap_sg_talitos_ptr(dev
, req_ctx
->psrc
, NULL
, 0, edesc
);
1593 /* When using hashctx-in, must unmap it. */
1594 if (from_talitos_ptr_len(&edesc
->desc
.ptr
[1], is_sec1
))
1595 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[1],
1598 if (from_talitos_ptr_len(&edesc
->desc
.ptr
[2], is_sec1
))
1599 unmap_single_talitos_ptr(dev
, &edesc
->desc
.ptr
[2],
1603 dma_unmap_single(dev
, edesc
->dma_link_tbl
, edesc
->dma_len
,
1608 static void ahash_done(struct device
*dev
,
1609 struct talitos_desc
*desc
, void *context
,
1612 struct ahash_request
*areq
= context
;
1613 struct talitos_edesc
*edesc
=
1614 container_of(desc
, struct talitos_edesc
, desc
);
1615 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1617 if (!req_ctx
->last
&& req_ctx
->to_hash_later
) {
1618 /* Position any partial block for next update/final/finup */
1619 memcpy(req_ctx
->buf
, req_ctx
->bufnext
, req_ctx
->to_hash_later
);
1620 req_ctx
->nbuf
= req_ctx
->to_hash_later
;
1622 common_nonsnoop_hash_unmap(dev
, edesc
, areq
);
1626 areq
->base
.complete(&areq
->base
, err
);
1629 static int common_nonsnoop_hash(struct talitos_edesc
*edesc
,
1630 struct ahash_request
*areq
, unsigned int length
,
1631 void (*callback
) (struct device
*dev
,
1632 struct talitos_desc
*desc
,
1633 void *context
, int error
))
1635 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(areq
);
1636 struct talitos_ctx
*ctx
= crypto_ahash_ctx(tfm
);
1637 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1638 struct device
*dev
= ctx
->dev
;
1639 struct talitos_desc
*desc
= &edesc
->desc
;
1641 struct talitos_private
*priv
= dev_get_drvdata(dev
);
1642 bool is_sec1
= has_ftr_sec1(priv
);
1644 /* first DWORD empty */
1645 desc
->ptr
[0] = zero_entry
;
1647 /* hash context in */
1648 if (!req_ctx
->first
|| req_ctx
->swinit
) {
1649 map_single_talitos_ptr(dev
, &desc
->ptr
[1],
1650 req_ctx
->hw_context_size
,
1651 (char *)req_ctx
->hw_context
,
1653 req_ctx
->swinit
= 0;
1655 desc
->ptr
[1] = zero_entry
;
1656 /* Indicate next op is not the first. */
1662 map_single_talitos_ptr(dev
, &desc
->ptr
[2], ctx
->keylen
,
1663 (char *)&ctx
->key
, DMA_TO_DEVICE
);
1665 desc
->ptr
[2] = zero_entry
;
1670 map_sg_in_talitos_ptr(dev
, req_ctx
->psrc
, length
, edesc
,
1671 DMA_TO_DEVICE
, &desc
->ptr
[3]);
1673 /* fifth DWORD empty */
1674 desc
->ptr
[4] = zero_entry
;
1676 /* hash/HMAC out -or- hash context out */
1678 map_single_talitos_ptr(dev
, &desc
->ptr
[5],
1679 crypto_ahash_digestsize(tfm
),
1680 areq
->result
, DMA_FROM_DEVICE
);
1682 map_single_talitos_ptr(dev
, &desc
->ptr
[5],
1683 req_ctx
->hw_context_size
,
1684 req_ctx
->hw_context
, DMA_FROM_DEVICE
);
1686 /* last DWORD empty */
1687 desc
->ptr
[6] = zero_entry
;
1689 ret
= talitos_submit(dev
, ctx
->ch
, desc
, callback
, areq
);
1690 if (ret
!= -EINPROGRESS
) {
1691 common_nonsnoop_hash_unmap(dev
, edesc
, areq
);
1697 static struct talitos_edesc
*ahash_edesc_alloc(struct ahash_request
*areq
,
1698 unsigned int nbytes
)
1700 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(areq
);
1701 struct talitos_ctx
*ctx
= crypto_ahash_ctx(tfm
);
1702 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1704 return talitos_edesc_alloc(ctx
->dev
, NULL
, req_ctx
->psrc
, NULL
, NULL
, 0,
1705 nbytes
, 0, 0, 0, areq
->base
.flags
, false);
1708 static int ahash_init(struct ahash_request
*areq
)
1710 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(areq
);
1711 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1713 /* Initialize the context */
1715 req_ctx
->first
= 1; /* first indicates h/w must init its context */
1716 req_ctx
->swinit
= 0; /* assume h/w init of context */
1717 req_ctx
->hw_context_size
=
1718 (crypto_ahash_digestsize(tfm
) <= SHA256_DIGEST_SIZE
)
1719 ? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
1720 : TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512
;
1726 * on h/w without explicit sha224 support, we initialize h/w context
1727 * manually with sha224 constants, and tell it to run sha256.
1729 static int ahash_init_sha224_swinit(struct ahash_request
*areq
)
1731 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1734 req_ctx
->swinit
= 1;/* prevent h/w initting context with sha256 values*/
1736 req_ctx
->hw_context
[0] = SHA224_H0
;
1737 req_ctx
->hw_context
[1] = SHA224_H1
;
1738 req_ctx
->hw_context
[2] = SHA224_H2
;
1739 req_ctx
->hw_context
[3] = SHA224_H3
;
1740 req_ctx
->hw_context
[4] = SHA224_H4
;
1741 req_ctx
->hw_context
[5] = SHA224_H5
;
1742 req_ctx
->hw_context
[6] = SHA224_H6
;
1743 req_ctx
->hw_context
[7] = SHA224_H7
;
1745 /* init 64-bit count */
1746 req_ctx
->hw_context
[8] = 0;
1747 req_ctx
->hw_context
[9] = 0;
1752 static int ahash_process_req(struct ahash_request
*areq
, unsigned int nbytes
)
1754 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(areq
);
1755 struct talitos_ctx
*ctx
= crypto_ahash_ctx(tfm
);
1756 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1757 struct talitos_edesc
*edesc
;
1758 unsigned int blocksize
=
1759 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm
));
1760 unsigned int nbytes_to_hash
;
1761 unsigned int to_hash_later
;
1765 if (!req_ctx
->last
&& (nbytes
+ req_ctx
->nbuf
<= blocksize
)) {
1766 /* Buffer up to one whole block */
1767 sg_copy_to_buffer(areq
->src
,
1768 sg_count(areq
->src
, nbytes
, &chained
),
1769 req_ctx
->buf
+ req_ctx
->nbuf
, nbytes
);
1770 req_ctx
->nbuf
+= nbytes
;
1774 /* At least (blocksize + 1) bytes are available to hash */
1775 nbytes_to_hash
= nbytes
+ req_ctx
->nbuf
;
1776 to_hash_later
= nbytes_to_hash
& (blocksize
- 1);
1780 else if (to_hash_later
)
1781 /* There is a partial block. Hash the full block(s) now */
1782 nbytes_to_hash
-= to_hash_later
;
1784 /* Keep one block buffered */
1785 nbytes_to_hash
-= blocksize
;
1786 to_hash_later
= blocksize
;
1789 /* Chain in any previously buffered data */
1790 if (req_ctx
->nbuf
) {
1791 nsg
= (req_ctx
->nbuf
< nbytes_to_hash
) ? 2 : 1;
1792 sg_init_table(req_ctx
->bufsl
, nsg
);
1793 sg_set_buf(req_ctx
->bufsl
, req_ctx
->buf
, req_ctx
->nbuf
);
1795 scatterwalk_sg_chain(req_ctx
->bufsl
, 2, areq
->src
);
1796 req_ctx
->psrc
= req_ctx
->bufsl
;
1798 req_ctx
->psrc
= areq
->src
;
1800 if (to_hash_later
) {
1801 int nents
= sg_count(areq
->src
, nbytes
, &chained
);
1802 sg_pcopy_to_buffer(areq
->src
, nents
,
1805 nbytes
- to_hash_later
);
1807 req_ctx
->to_hash_later
= to_hash_later
;
1809 /* Allocate extended descriptor */
1810 edesc
= ahash_edesc_alloc(areq
, nbytes_to_hash
);
1812 return PTR_ERR(edesc
);
1814 edesc
->desc
.hdr
= ctx
->desc_hdr_template
;
1816 /* On last one, request SEC to pad; otherwise continue */
1818 edesc
->desc
.hdr
|= DESC_HDR_MODE0_MDEU_PAD
;
1820 edesc
->desc
.hdr
|= DESC_HDR_MODE0_MDEU_CONT
;
1822 /* request SEC to INIT hash. */
1823 if (req_ctx
->first
&& !req_ctx
->swinit
)
1824 edesc
->desc
.hdr
|= DESC_HDR_MODE0_MDEU_INIT
;
1826 /* When the tfm context has a keylen, it's an HMAC.
1827 * A first or last (ie. not middle) descriptor must request HMAC.
1829 if (ctx
->keylen
&& (req_ctx
->first
|| req_ctx
->last
))
1830 edesc
->desc
.hdr
|= DESC_HDR_MODE0_MDEU_HMAC
;
1832 return common_nonsnoop_hash(edesc
, areq
, nbytes_to_hash
,
1836 static int ahash_update(struct ahash_request
*areq
)
1838 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1842 return ahash_process_req(areq
, areq
->nbytes
);
1845 static int ahash_final(struct ahash_request
*areq
)
1847 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1851 return ahash_process_req(areq
, 0);
1854 static int ahash_finup(struct ahash_request
*areq
)
1856 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1860 return ahash_process_req(areq
, areq
->nbytes
);
1863 static int ahash_digest(struct ahash_request
*areq
)
1865 struct talitos_ahash_req_ctx
*req_ctx
= ahash_request_ctx(areq
);
1866 struct crypto_ahash
*ahash
= crypto_ahash_reqtfm(areq
);
1871 return ahash_process_req(areq
, areq
->nbytes
);
1874 struct keyhash_result
{
1875 struct completion completion
;
1879 static void keyhash_complete(struct crypto_async_request
*req
, int err
)
1881 struct keyhash_result
*res
= req
->data
;
1883 if (err
== -EINPROGRESS
)
1887 complete(&res
->completion
);
1890 static int keyhash(struct crypto_ahash
*tfm
, const u8
*key
, unsigned int keylen
,
1893 struct talitos_ctx
*ctx
= crypto_tfm_ctx(crypto_ahash_tfm(tfm
));
1895 struct scatterlist sg
[1];
1896 struct ahash_request
*req
;
1897 struct keyhash_result hresult
;
1900 init_completion(&hresult
.completion
);
1902 req
= ahash_request_alloc(tfm
, GFP_KERNEL
);
1906 /* Keep tfm keylen == 0 during hash of the long key */
1908 ahash_request_set_callback(req
, CRYPTO_TFM_REQ_MAY_BACKLOG
,
1909 keyhash_complete
, &hresult
);
1911 sg_init_one(&sg
[0], key
, keylen
);
1913 ahash_request_set_crypt(req
, sg
, hash
, keylen
);
1914 ret
= crypto_ahash_digest(req
);
1920 ret
= wait_for_completion_interruptible(
1921 &hresult
.completion
);
1928 ahash_request_free(req
);
1933 static int ahash_setkey(struct crypto_ahash
*tfm
, const u8
*key
,
1934 unsigned int keylen
)
1936 struct talitos_ctx
*ctx
= crypto_tfm_ctx(crypto_ahash_tfm(tfm
));
1937 unsigned int blocksize
=
1938 crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm
));
1939 unsigned int digestsize
= crypto_ahash_digestsize(tfm
);
1940 unsigned int keysize
= keylen
;
1941 u8 hash
[SHA512_DIGEST_SIZE
];
1944 if (keylen
<= blocksize
)
1945 memcpy(ctx
->key
, key
, keysize
);
1947 /* Must get the hash of the long key */
1948 ret
= keyhash(tfm
, key
, keylen
, hash
);
1951 crypto_ahash_set_flags(tfm
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
1955 keysize
= digestsize
;
1956 memcpy(ctx
->key
, hash
, digestsize
);
1959 ctx
->keylen
= keysize
;
1965 struct talitos_alg_template
{
1968 struct crypto_alg crypto
;
1969 struct ahash_alg hash
;
1971 __be32 desc_hdr_template
;
1974 static struct talitos_alg_template driver_algs
[] = {
1975 /* AEAD algorithms. These use a single-pass ipsec_esp descriptor */
1976 { .type
= CRYPTO_ALG_TYPE_AEAD
,
1978 .cra_name
= "authenc(hmac(sha1),cbc(aes))",
1979 .cra_driver_name
= "authenc-hmac-sha1-cbc-aes-talitos",
1980 .cra_blocksize
= AES_BLOCK_SIZE
,
1981 .cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
,
1983 .ivsize
= AES_BLOCK_SIZE
,
1984 .maxauthsize
= SHA1_DIGEST_SIZE
,
1987 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
1988 DESC_HDR_SEL0_AESU
|
1989 DESC_HDR_MODE0_AESU_CBC
|
1990 DESC_HDR_SEL1_MDEUA
|
1991 DESC_HDR_MODE1_MDEU_INIT
|
1992 DESC_HDR_MODE1_MDEU_PAD
|
1993 DESC_HDR_MODE1_MDEU_SHA1_HMAC
,
1995 { .type
= CRYPTO_ALG_TYPE_AEAD
,
1997 .cra_name
= "authenc(hmac(sha1),cbc(des3_ede))",
1998 .cra_driver_name
= "authenc-hmac-sha1-cbc-3des-talitos",
1999 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2000 .cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
,
2002 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2003 .maxauthsize
= SHA1_DIGEST_SIZE
,
2006 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2008 DESC_HDR_MODE0_DEU_CBC
|
2009 DESC_HDR_MODE0_DEU_3DES
|
2010 DESC_HDR_SEL1_MDEUA
|
2011 DESC_HDR_MODE1_MDEU_INIT
|
2012 DESC_HDR_MODE1_MDEU_PAD
|
2013 DESC_HDR_MODE1_MDEU_SHA1_HMAC
,
2015 { .type
= CRYPTO_ALG_TYPE_AEAD
,
2017 .cra_name
= "authenc(hmac(sha224),cbc(aes))",
2018 .cra_driver_name
= "authenc-hmac-sha224-cbc-aes-talitos",
2019 .cra_blocksize
= AES_BLOCK_SIZE
,
2020 .cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
,
2022 .ivsize
= AES_BLOCK_SIZE
,
2023 .maxauthsize
= SHA224_DIGEST_SIZE
,
2026 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2027 DESC_HDR_SEL0_AESU
|
2028 DESC_HDR_MODE0_AESU_CBC
|
2029 DESC_HDR_SEL1_MDEUA
|
2030 DESC_HDR_MODE1_MDEU_INIT
|
2031 DESC_HDR_MODE1_MDEU_PAD
|
2032 DESC_HDR_MODE1_MDEU_SHA224_HMAC
,
2034 { .type
= CRYPTO_ALG_TYPE_AEAD
,
2036 .cra_name
= "authenc(hmac(sha224),cbc(des3_ede))",
2037 .cra_driver_name
= "authenc-hmac-sha224-cbc-3des-talitos",
2038 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2039 .cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
,
2041 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2042 .maxauthsize
= SHA224_DIGEST_SIZE
,
2045 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2047 DESC_HDR_MODE0_DEU_CBC
|
2048 DESC_HDR_MODE0_DEU_3DES
|
2049 DESC_HDR_SEL1_MDEUA
|
2050 DESC_HDR_MODE1_MDEU_INIT
|
2051 DESC_HDR_MODE1_MDEU_PAD
|
2052 DESC_HDR_MODE1_MDEU_SHA224_HMAC
,
2054 { .type
= CRYPTO_ALG_TYPE_AEAD
,
2056 .cra_name
= "authenc(hmac(sha256),cbc(aes))",
2057 .cra_driver_name
= "authenc-hmac-sha256-cbc-aes-talitos",
2058 .cra_blocksize
= AES_BLOCK_SIZE
,
2059 .cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
,
2061 .ivsize
= AES_BLOCK_SIZE
,
2062 .maxauthsize
= SHA256_DIGEST_SIZE
,
2065 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2066 DESC_HDR_SEL0_AESU
|
2067 DESC_HDR_MODE0_AESU_CBC
|
2068 DESC_HDR_SEL1_MDEUA
|
2069 DESC_HDR_MODE1_MDEU_INIT
|
2070 DESC_HDR_MODE1_MDEU_PAD
|
2071 DESC_HDR_MODE1_MDEU_SHA256_HMAC
,
2073 { .type
= CRYPTO_ALG_TYPE_AEAD
,
2075 .cra_name
= "authenc(hmac(sha256),cbc(des3_ede))",
2076 .cra_driver_name
= "authenc-hmac-sha256-cbc-3des-talitos",
2077 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2078 .cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
,
2080 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2081 .maxauthsize
= SHA256_DIGEST_SIZE
,
2084 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2086 DESC_HDR_MODE0_DEU_CBC
|
2087 DESC_HDR_MODE0_DEU_3DES
|
2088 DESC_HDR_SEL1_MDEUA
|
2089 DESC_HDR_MODE1_MDEU_INIT
|
2090 DESC_HDR_MODE1_MDEU_PAD
|
2091 DESC_HDR_MODE1_MDEU_SHA256_HMAC
,
2093 { .type
= CRYPTO_ALG_TYPE_AEAD
,
2095 .cra_name
= "authenc(hmac(sha384),cbc(aes))",
2096 .cra_driver_name
= "authenc-hmac-sha384-cbc-aes-talitos",
2097 .cra_blocksize
= AES_BLOCK_SIZE
,
2098 .cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
,
2100 .ivsize
= AES_BLOCK_SIZE
,
2101 .maxauthsize
= SHA384_DIGEST_SIZE
,
2104 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2105 DESC_HDR_SEL0_AESU
|
2106 DESC_HDR_MODE0_AESU_CBC
|
2107 DESC_HDR_SEL1_MDEUB
|
2108 DESC_HDR_MODE1_MDEU_INIT
|
2109 DESC_HDR_MODE1_MDEU_PAD
|
2110 DESC_HDR_MODE1_MDEUB_SHA384_HMAC
,
2112 { .type
= CRYPTO_ALG_TYPE_AEAD
,
2114 .cra_name
= "authenc(hmac(sha384),cbc(des3_ede))",
2115 .cra_driver_name
= "authenc-hmac-sha384-cbc-3des-talitos",
2116 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2117 .cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
,
2119 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2120 .maxauthsize
= SHA384_DIGEST_SIZE
,
2123 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2125 DESC_HDR_MODE0_DEU_CBC
|
2126 DESC_HDR_MODE0_DEU_3DES
|
2127 DESC_HDR_SEL1_MDEUB
|
2128 DESC_HDR_MODE1_MDEU_INIT
|
2129 DESC_HDR_MODE1_MDEU_PAD
|
2130 DESC_HDR_MODE1_MDEUB_SHA384_HMAC
,
2132 { .type
= CRYPTO_ALG_TYPE_AEAD
,
2134 .cra_name
= "authenc(hmac(sha512),cbc(aes))",
2135 .cra_driver_name
= "authenc-hmac-sha512-cbc-aes-talitos",
2136 .cra_blocksize
= AES_BLOCK_SIZE
,
2137 .cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
,
2139 .ivsize
= AES_BLOCK_SIZE
,
2140 .maxauthsize
= SHA512_DIGEST_SIZE
,
2143 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2144 DESC_HDR_SEL0_AESU
|
2145 DESC_HDR_MODE0_AESU_CBC
|
2146 DESC_HDR_SEL1_MDEUB
|
2147 DESC_HDR_MODE1_MDEU_INIT
|
2148 DESC_HDR_MODE1_MDEU_PAD
|
2149 DESC_HDR_MODE1_MDEUB_SHA512_HMAC
,
2151 { .type
= CRYPTO_ALG_TYPE_AEAD
,
2153 .cra_name
= "authenc(hmac(sha512),cbc(des3_ede))",
2154 .cra_driver_name
= "authenc-hmac-sha512-cbc-3des-talitos",
2155 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2156 .cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
,
2158 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2159 .maxauthsize
= SHA512_DIGEST_SIZE
,
2162 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2164 DESC_HDR_MODE0_DEU_CBC
|
2165 DESC_HDR_MODE0_DEU_3DES
|
2166 DESC_HDR_SEL1_MDEUB
|
2167 DESC_HDR_MODE1_MDEU_INIT
|
2168 DESC_HDR_MODE1_MDEU_PAD
|
2169 DESC_HDR_MODE1_MDEUB_SHA512_HMAC
,
2171 { .type
= CRYPTO_ALG_TYPE_AEAD
,
2173 .cra_name
= "authenc(hmac(md5),cbc(aes))",
2174 .cra_driver_name
= "authenc-hmac-md5-cbc-aes-talitos",
2175 .cra_blocksize
= AES_BLOCK_SIZE
,
2176 .cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
,
2178 .ivsize
= AES_BLOCK_SIZE
,
2179 .maxauthsize
= MD5_DIGEST_SIZE
,
2182 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2183 DESC_HDR_SEL0_AESU
|
2184 DESC_HDR_MODE0_AESU_CBC
|
2185 DESC_HDR_SEL1_MDEUA
|
2186 DESC_HDR_MODE1_MDEU_INIT
|
2187 DESC_HDR_MODE1_MDEU_PAD
|
2188 DESC_HDR_MODE1_MDEU_MD5_HMAC
,
2190 { .type
= CRYPTO_ALG_TYPE_AEAD
,
2192 .cra_name
= "authenc(hmac(md5),cbc(des3_ede))",
2193 .cra_driver_name
= "authenc-hmac-md5-cbc-3des-talitos",
2194 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2195 .cra_flags
= CRYPTO_ALG_TYPE_AEAD
| CRYPTO_ALG_ASYNC
,
2197 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2198 .maxauthsize
= MD5_DIGEST_SIZE
,
2201 .desc_hdr_template
= DESC_HDR_TYPE_IPSEC_ESP
|
2203 DESC_HDR_MODE0_DEU_CBC
|
2204 DESC_HDR_MODE0_DEU_3DES
|
2205 DESC_HDR_SEL1_MDEUA
|
2206 DESC_HDR_MODE1_MDEU_INIT
|
2207 DESC_HDR_MODE1_MDEU_PAD
|
2208 DESC_HDR_MODE1_MDEU_MD5_HMAC
,
2210 /* ABLKCIPHER algorithms. */
2211 { .type
= CRYPTO_ALG_TYPE_ABLKCIPHER
,
2213 .cra_name
= "cbc(aes)",
2214 .cra_driver_name
= "cbc-aes-talitos",
2215 .cra_blocksize
= AES_BLOCK_SIZE
,
2216 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|
2219 .min_keysize
= AES_MIN_KEY_SIZE
,
2220 .max_keysize
= AES_MAX_KEY_SIZE
,
2221 .ivsize
= AES_BLOCK_SIZE
,
2224 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2225 DESC_HDR_SEL0_AESU
|
2226 DESC_HDR_MODE0_AESU_CBC
,
2228 { .type
= CRYPTO_ALG_TYPE_ABLKCIPHER
,
2230 .cra_name
= "cbc(des3_ede)",
2231 .cra_driver_name
= "cbc-3des-talitos",
2232 .cra_blocksize
= DES3_EDE_BLOCK_SIZE
,
2233 .cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|
2236 .min_keysize
= DES3_EDE_KEY_SIZE
,
2237 .max_keysize
= DES3_EDE_KEY_SIZE
,
2238 .ivsize
= DES3_EDE_BLOCK_SIZE
,
2241 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2243 DESC_HDR_MODE0_DEU_CBC
|
2244 DESC_HDR_MODE0_DEU_3DES
,
2246 /* AHASH algorithms. */
2247 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2249 .halg
.digestsize
= MD5_DIGEST_SIZE
,
2252 .cra_driver_name
= "md5-talitos",
2253 .cra_blocksize
= MD5_HMAC_BLOCK_SIZE
,
2254 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2258 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2259 DESC_HDR_SEL0_MDEUA
|
2260 DESC_HDR_MODE0_MDEU_MD5
,
2262 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2264 .halg
.digestsize
= SHA1_DIGEST_SIZE
,
2267 .cra_driver_name
= "sha1-talitos",
2268 .cra_blocksize
= SHA1_BLOCK_SIZE
,
2269 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2273 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2274 DESC_HDR_SEL0_MDEUA
|
2275 DESC_HDR_MODE0_MDEU_SHA1
,
2277 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2279 .halg
.digestsize
= SHA224_DIGEST_SIZE
,
2281 .cra_name
= "sha224",
2282 .cra_driver_name
= "sha224-talitos",
2283 .cra_blocksize
= SHA224_BLOCK_SIZE
,
2284 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2288 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2289 DESC_HDR_SEL0_MDEUA
|
2290 DESC_HDR_MODE0_MDEU_SHA224
,
2292 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2294 .halg
.digestsize
= SHA256_DIGEST_SIZE
,
2296 .cra_name
= "sha256",
2297 .cra_driver_name
= "sha256-talitos",
2298 .cra_blocksize
= SHA256_BLOCK_SIZE
,
2299 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2303 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2304 DESC_HDR_SEL0_MDEUA
|
2305 DESC_HDR_MODE0_MDEU_SHA256
,
2307 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2309 .halg
.digestsize
= SHA384_DIGEST_SIZE
,
2311 .cra_name
= "sha384",
2312 .cra_driver_name
= "sha384-talitos",
2313 .cra_blocksize
= SHA384_BLOCK_SIZE
,
2314 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2318 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2319 DESC_HDR_SEL0_MDEUB
|
2320 DESC_HDR_MODE0_MDEUB_SHA384
,
2322 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2324 .halg
.digestsize
= SHA512_DIGEST_SIZE
,
2326 .cra_name
= "sha512",
2327 .cra_driver_name
= "sha512-talitos",
2328 .cra_blocksize
= SHA512_BLOCK_SIZE
,
2329 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2333 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2334 DESC_HDR_SEL0_MDEUB
|
2335 DESC_HDR_MODE0_MDEUB_SHA512
,
2337 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2339 .halg
.digestsize
= MD5_DIGEST_SIZE
,
2341 .cra_name
= "hmac(md5)",
2342 .cra_driver_name
= "hmac-md5-talitos",
2343 .cra_blocksize
= MD5_HMAC_BLOCK_SIZE
,
2344 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2348 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2349 DESC_HDR_SEL0_MDEUA
|
2350 DESC_HDR_MODE0_MDEU_MD5
,
2352 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2354 .halg
.digestsize
= SHA1_DIGEST_SIZE
,
2356 .cra_name
= "hmac(sha1)",
2357 .cra_driver_name
= "hmac-sha1-talitos",
2358 .cra_blocksize
= SHA1_BLOCK_SIZE
,
2359 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2363 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2364 DESC_HDR_SEL0_MDEUA
|
2365 DESC_HDR_MODE0_MDEU_SHA1
,
2367 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2369 .halg
.digestsize
= SHA224_DIGEST_SIZE
,
2371 .cra_name
= "hmac(sha224)",
2372 .cra_driver_name
= "hmac-sha224-talitos",
2373 .cra_blocksize
= SHA224_BLOCK_SIZE
,
2374 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2378 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2379 DESC_HDR_SEL0_MDEUA
|
2380 DESC_HDR_MODE0_MDEU_SHA224
,
2382 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2384 .halg
.digestsize
= SHA256_DIGEST_SIZE
,
2386 .cra_name
= "hmac(sha256)",
2387 .cra_driver_name
= "hmac-sha256-talitos",
2388 .cra_blocksize
= SHA256_BLOCK_SIZE
,
2389 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2393 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2394 DESC_HDR_SEL0_MDEUA
|
2395 DESC_HDR_MODE0_MDEU_SHA256
,
2397 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2399 .halg
.digestsize
= SHA384_DIGEST_SIZE
,
2401 .cra_name
= "hmac(sha384)",
2402 .cra_driver_name
= "hmac-sha384-talitos",
2403 .cra_blocksize
= SHA384_BLOCK_SIZE
,
2404 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2408 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2409 DESC_HDR_SEL0_MDEUB
|
2410 DESC_HDR_MODE0_MDEUB_SHA384
,
2412 { .type
= CRYPTO_ALG_TYPE_AHASH
,
2414 .halg
.digestsize
= SHA512_DIGEST_SIZE
,
2416 .cra_name
= "hmac(sha512)",
2417 .cra_driver_name
= "hmac-sha512-talitos",
2418 .cra_blocksize
= SHA512_BLOCK_SIZE
,
2419 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
|
2423 .desc_hdr_template
= DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2424 DESC_HDR_SEL0_MDEUB
|
2425 DESC_HDR_MODE0_MDEUB_SHA512
,
2429 struct talitos_crypto_alg
{
2430 struct list_head entry
;
2432 struct talitos_alg_template algt
;
2435 static int talitos_cra_init(struct crypto_tfm
*tfm
)
2437 struct crypto_alg
*alg
= tfm
->__crt_alg
;
2438 struct talitos_crypto_alg
*talitos_alg
;
2439 struct talitos_ctx
*ctx
= crypto_tfm_ctx(tfm
);
2440 struct talitos_private
*priv
;
2442 if ((alg
->cra_flags
& CRYPTO_ALG_TYPE_MASK
) == CRYPTO_ALG_TYPE_AHASH
)
2443 talitos_alg
= container_of(__crypto_ahash_alg(alg
),
2444 struct talitos_crypto_alg
,
2447 talitos_alg
= container_of(alg
, struct talitos_crypto_alg
,
2450 /* update context with ptr to dev */
2451 ctx
->dev
= talitos_alg
->dev
;
2453 /* assign SEC channel to tfm in round-robin fashion */
2454 priv
= dev_get_drvdata(ctx
->dev
);
2455 ctx
->ch
= atomic_inc_return(&priv
->last_chan
) &
2456 (priv
->num_channels
- 1);
2458 /* copy descriptor header template value */
2459 ctx
->desc_hdr_template
= talitos_alg
->algt
.desc_hdr_template
;
2461 /* select done notification */
2462 ctx
->desc_hdr_template
|= DESC_HDR_DONE_NOTIFY
;
2467 static int talitos_cra_init_aead(struct crypto_tfm
*tfm
)
2469 struct talitos_ctx
*ctx
= crypto_tfm_ctx(tfm
);
2471 talitos_cra_init(tfm
);
2473 /* random first IV */
2474 get_random_bytes(ctx
->iv
, TALITOS_MAX_IV_LENGTH
);
2479 static int talitos_cra_init_ahash(struct crypto_tfm
*tfm
)
2481 struct talitos_ctx
*ctx
= crypto_tfm_ctx(tfm
);
2483 talitos_cra_init(tfm
);
2486 crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm
),
2487 sizeof(struct talitos_ahash_req_ctx
));
2493 * given the alg's descriptor header template, determine whether descriptor
2494 * type and primary/secondary execution units required match the hw
2495 * capabilities description provided in the device tree node.
2497 static int hw_supports(struct device
*dev
, __be32 desc_hdr_template
)
2499 struct talitos_private
*priv
= dev_get_drvdata(dev
);
2502 ret
= (1 << DESC_TYPE(desc_hdr_template
) & priv
->desc_types
) &&
2503 (1 << PRIMARY_EU(desc_hdr_template
) & priv
->exec_units
);
2505 if (SECONDARY_EU(desc_hdr_template
))
2506 ret
= ret
&& (1 << SECONDARY_EU(desc_hdr_template
)
2507 & priv
->exec_units
);
2512 static int talitos_remove(struct platform_device
*ofdev
)
2514 struct device
*dev
= &ofdev
->dev
;
2515 struct talitos_private
*priv
= dev_get_drvdata(dev
);
2516 struct talitos_crypto_alg
*t_alg
, *n
;
2519 list_for_each_entry_safe(t_alg
, n
, &priv
->alg_list
, entry
) {
2520 switch (t_alg
->algt
.type
) {
2521 case CRYPTO_ALG_TYPE_ABLKCIPHER
:
2522 case CRYPTO_ALG_TYPE_AEAD
:
2523 crypto_unregister_alg(&t_alg
->algt
.alg
.crypto
);
2525 case CRYPTO_ALG_TYPE_AHASH
:
2526 crypto_unregister_ahash(&t_alg
->algt
.alg
.hash
);
2529 list_del(&t_alg
->entry
);
2533 if (hw_supports(dev
, DESC_HDR_SEL0_RNG
))
2534 talitos_unregister_rng(dev
);
2536 for (i
= 0; i
< priv
->num_channels
; i
++)
2537 kfree(priv
->chan
[i
].fifo
);
2541 for (i
= 0; i
< 2; i
++)
2543 free_irq(priv
->irq
[i
], dev
);
2544 irq_dispose_mapping(priv
->irq
[i
]);
2547 tasklet_kill(&priv
->done_task
[0]);
2549 tasklet_kill(&priv
->done_task
[1]);
2558 static struct talitos_crypto_alg
*talitos_alg_alloc(struct device
*dev
,
2559 struct talitos_alg_template
2562 struct talitos_private
*priv
= dev_get_drvdata(dev
);
2563 struct talitos_crypto_alg
*t_alg
;
2564 struct crypto_alg
*alg
;
2566 t_alg
= kzalloc(sizeof(struct talitos_crypto_alg
), GFP_KERNEL
);
2568 return ERR_PTR(-ENOMEM
);
2570 t_alg
->algt
= *template;
2572 switch (t_alg
->algt
.type
) {
2573 case CRYPTO_ALG_TYPE_ABLKCIPHER
:
2574 alg
= &t_alg
->algt
.alg
.crypto
;
2575 alg
->cra_init
= talitos_cra_init
;
2576 alg
->cra_type
= &crypto_ablkcipher_type
;
2577 alg
->cra_ablkcipher
.setkey
= ablkcipher_setkey
;
2578 alg
->cra_ablkcipher
.encrypt
= ablkcipher_encrypt
;
2579 alg
->cra_ablkcipher
.decrypt
= ablkcipher_decrypt
;
2580 alg
->cra_ablkcipher
.geniv
= "eseqiv";
2582 case CRYPTO_ALG_TYPE_AEAD
:
2583 alg
= &t_alg
->algt
.alg
.crypto
;
2584 alg
->cra_init
= talitos_cra_init_aead
;
2585 alg
->cra_type
= &crypto_aead_type
;
2586 alg
->cra_aead
.setkey
= aead_setkey
;
2587 alg
->cra_aead
.setauthsize
= aead_setauthsize
;
2588 alg
->cra_aead
.encrypt
= aead_encrypt
;
2589 alg
->cra_aead
.decrypt
= aead_decrypt
;
2590 alg
->cra_aead
.givencrypt
= aead_givencrypt
;
2591 alg
->cra_aead
.geniv
= "<built-in>";
2593 case CRYPTO_ALG_TYPE_AHASH
:
2594 alg
= &t_alg
->algt
.alg
.hash
.halg
.base
;
2595 alg
->cra_init
= talitos_cra_init_ahash
;
2596 alg
->cra_type
= &crypto_ahash_type
;
2597 t_alg
->algt
.alg
.hash
.init
= ahash_init
;
2598 t_alg
->algt
.alg
.hash
.update
= ahash_update
;
2599 t_alg
->algt
.alg
.hash
.final
= ahash_final
;
2600 t_alg
->algt
.alg
.hash
.finup
= ahash_finup
;
2601 t_alg
->algt
.alg
.hash
.digest
= ahash_digest
;
2602 t_alg
->algt
.alg
.hash
.setkey
= ahash_setkey
;
2604 if (!(priv
->features
& TALITOS_FTR_HMAC_OK
) &&
2605 !strncmp(alg
->cra_name
, "hmac", 4)) {
2607 return ERR_PTR(-ENOTSUPP
);
2609 if (!(priv
->features
& TALITOS_FTR_SHA224_HWINIT
) &&
2610 (!strcmp(alg
->cra_name
, "sha224") ||
2611 !strcmp(alg
->cra_name
, "hmac(sha224)"))) {
2612 t_alg
->algt
.alg
.hash
.init
= ahash_init_sha224_swinit
;
2613 t_alg
->algt
.desc_hdr_template
=
2614 DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU
|
2615 DESC_HDR_SEL0_MDEUA
|
2616 DESC_HDR_MODE0_MDEU_SHA256
;
2620 dev_err(dev
, "unknown algorithm type %d\n", t_alg
->algt
.type
);
2621 return ERR_PTR(-EINVAL
);
2624 alg
->cra_module
= THIS_MODULE
;
2625 alg
->cra_priority
= TALITOS_CRA_PRIORITY
;
2626 alg
->cra_alignmask
= 0;
2627 alg
->cra_ctxsize
= sizeof(struct talitos_ctx
);
2628 alg
->cra_flags
|= CRYPTO_ALG_KERN_DRIVER_ONLY
;
2635 static int talitos_probe_irq(struct platform_device
*ofdev
)
2637 struct device
*dev
= &ofdev
->dev
;
2638 struct device_node
*np
= ofdev
->dev
.of_node
;
2639 struct talitos_private
*priv
= dev_get_drvdata(dev
);
2642 priv
->irq
[0] = irq_of_parse_and_map(np
, 0);
2643 if (!priv
->irq
[0]) {
2644 dev_err(dev
, "failed to map irq\n");
2648 priv
->irq
[1] = irq_of_parse_and_map(np
, 1);
2650 /* get the primary irq line */
2651 if (!priv
->irq
[1]) {
2652 err
= request_irq(priv
->irq
[0], talitos_interrupt_4ch
, 0,
2653 dev_driver_string(dev
), dev
);
2657 err
= request_irq(priv
->irq
[0], talitos_interrupt_ch0_2
, 0,
2658 dev_driver_string(dev
), dev
);
2662 /* get the secondary irq line */
2663 err
= request_irq(priv
->irq
[1], talitos_interrupt_ch1_3
, 0,
2664 dev_driver_string(dev
), dev
);
2666 dev_err(dev
, "failed to request secondary irq\n");
2667 irq_dispose_mapping(priv
->irq
[1]);
2675 dev_err(dev
, "failed to request primary irq\n");
2676 irq_dispose_mapping(priv
->irq
[0]);
2683 static int talitos_probe(struct platform_device
*ofdev
)
2685 struct device
*dev
= &ofdev
->dev
;
2686 struct device_node
*np
= ofdev
->dev
.of_node
;
2687 struct talitos_private
*priv
;
2688 const unsigned int *prop
;
2691 priv
= kzalloc(sizeof(struct talitos_private
), GFP_KERNEL
);
2695 INIT_LIST_HEAD(&priv
->alg_list
);
2697 dev_set_drvdata(dev
, priv
);
2699 priv
->ofdev
= ofdev
;
2701 spin_lock_init(&priv
->reg_lock
);
2703 err
= talitos_probe_irq(ofdev
);
2707 if (!priv
->irq
[1]) {
2708 tasklet_init(&priv
->done_task
[0], talitos_done_4ch
,
2709 (unsigned long)dev
);
2711 tasklet_init(&priv
->done_task
[0], talitos_done_ch0_2
,
2712 (unsigned long)dev
);
2713 tasklet_init(&priv
->done_task
[1], talitos_done_ch1_3
,
2714 (unsigned long)dev
);
2717 priv
->reg
= of_iomap(np
, 0);
2719 dev_err(dev
, "failed to of_iomap\n");
2724 /* get SEC version capabilities from device tree */
2725 prop
= of_get_property(np
, "fsl,num-channels", NULL
);
2727 priv
->num_channels
= *prop
;
2729 prop
= of_get_property(np
, "fsl,channel-fifo-len", NULL
);
2731 priv
->chfifo_len
= *prop
;
2733 prop
= of_get_property(np
, "fsl,exec-units-mask", NULL
);
2735 priv
->exec_units
= *prop
;
2737 prop
= of_get_property(np
, "fsl,descriptor-types-mask", NULL
);
2739 priv
->desc_types
= *prop
;
2741 if (!is_power_of_2(priv
->num_channels
) || !priv
->chfifo_len
||
2742 !priv
->exec_units
|| !priv
->desc_types
) {
2743 dev_err(dev
, "invalid property data in device tree node\n");
2748 if (of_device_is_compatible(np
, "fsl,sec3.0"))
2749 priv
->features
|= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT
;
2751 if (of_device_is_compatible(np
, "fsl,sec2.1"))
2752 priv
->features
|= TALITOS_FTR_HW_AUTH_CHECK
|
2753 TALITOS_FTR_SHA224_HWINIT
|
2754 TALITOS_FTR_HMAC_OK
;
2756 if (of_device_is_compatible(np
, "fsl,sec1.0"))
2757 priv
->features
|= TALITOS_FTR_SEC1
;
2759 priv
->chan
= kzalloc(sizeof(struct talitos_channel
) *
2760 priv
->num_channels
, GFP_KERNEL
);
2762 dev_err(dev
, "failed to allocate channel management space\n");
2767 priv
->fifo_len
= roundup_pow_of_two(priv
->chfifo_len
);
2769 for (i
= 0; i
< priv
->num_channels
; i
++) {
2770 priv
->chan
[i
].reg
= priv
->reg
+ TALITOS_CH_STRIDE
* (i
+ 1);
2771 if (!priv
->irq
[1] || !(i
& 1))
2772 priv
->chan
[i
].reg
+= TALITOS_CH_BASE_OFFSET
;
2774 spin_lock_init(&priv
->chan
[i
].head_lock
);
2775 spin_lock_init(&priv
->chan
[i
].tail_lock
);
2777 priv
->chan
[i
].fifo
= kzalloc(sizeof(struct talitos_request
) *
2778 priv
->fifo_len
, GFP_KERNEL
);
2779 if (!priv
->chan
[i
].fifo
) {
2780 dev_err(dev
, "failed to allocate request fifo %d\n", i
);
2785 atomic_set(&priv
->chan
[i
].submit_count
,
2786 -(priv
->chfifo_len
- 1));
2789 dma_set_mask(dev
, DMA_BIT_MASK(36));
2791 /* reset and initialize the h/w */
2792 err
= init_device(dev
);
2794 dev_err(dev
, "failed to initialize device\n");
2798 /* register the RNG, if available */
2799 if (hw_supports(dev
, DESC_HDR_SEL0_RNG
)) {
2800 err
= talitos_register_rng(dev
);
2802 dev_err(dev
, "failed to register hwrng: %d\n", err
);
2805 dev_info(dev
, "hwrng\n");
2808 /* register crypto algorithms the device supports */
2809 for (i
= 0; i
< ARRAY_SIZE(driver_algs
); i
++) {
2810 if (hw_supports(dev
, driver_algs
[i
].desc_hdr_template
)) {
2811 struct talitos_crypto_alg
*t_alg
;
2814 t_alg
= talitos_alg_alloc(dev
, &driver_algs
[i
]);
2815 if (IS_ERR(t_alg
)) {
2816 err
= PTR_ERR(t_alg
);
2817 if (err
== -ENOTSUPP
)
2822 switch (t_alg
->algt
.type
) {
2823 case CRYPTO_ALG_TYPE_ABLKCIPHER
:
2824 case CRYPTO_ALG_TYPE_AEAD
:
2825 err
= crypto_register_alg(
2826 &t_alg
->algt
.alg
.crypto
);
2827 name
= t_alg
->algt
.alg
.crypto
.cra_driver_name
;
2829 case CRYPTO_ALG_TYPE_AHASH
:
2830 err
= crypto_register_ahash(
2831 &t_alg
->algt
.alg
.hash
);
2833 t_alg
->algt
.alg
.hash
.halg
.base
.cra_driver_name
;
2837 dev_err(dev
, "%s alg registration failed\n",
2841 list_add_tail(&t_alg
->entry
, &priv
->alg_list
);
2844 if (!list_empty(&priv
->alg_list
))
2845 dev_info(dev
, "%s algorithms registered in /proc/crypto\n",
2846 (char *)of_get_property(np
, "compatible", NULL
));
2851 talitos_remove(ofdev
);
2856 static const struct of_device_id talitos_match
[] = {
2858 .compatible
= "fsl,sec2.0",
2862 MODULE_DEVICE_TABLE(of
, talitos_match
);
2864 static struct platform_driver talitos_driver
= {
2867 .of_match_table
= talitos_match
,
2869 .probe
= talitos_probe
,
2870 .remove
= talitos_remove
,
2873 module_platform_driver(talitos_driver
);
2875 MODULE_LICENSE("GPL");
2876 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
2877 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");