1 /* n2_core.c: Niagara2 Stream Processing Unit (SPU) crypto support.
3 * Copyright (C) 2010 David S. Miller <davem@davemloft.net>
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8 #include <linux/kernel.h>
9 #include <linux/module.h>
11 #include <linux/of_device.h>
12 #include <linux/cpumask.h>
13 #include <linux/slab.h>
14 #include <linux/interrupt.h>
15 #include <linux/crypto.h>
16 #include <crypto/md5.h>
17 #include <crypto/sha.h>
18 #include <crypto/aes.h>
19 #include <crypto/des.h>
20 #include <linux/mutex.h>
21 #include <linux/delay.h>
22 #include <linux/sched.h>
24 #include <crypto/internal/hash.h>
25 #include <crypto/scatterwalk.h>
26 #include <crypto/algapi.h>
28 #include <asm/hypervisor.h>
29 #include <asm/mdesc.h>
33 #define DRV_MODULE_NAME "n2_crypto"
34 #define DRV_MODULE_VERSION "0.1"
35 #define DRV_MODULE_RELDATE "April 29, 2010"
37 static char version
[] __devinitdata
=
38 DRV_MODULE_NAME
".c:v" DRV_MODULE_VERSION
" (" DRV_MODULE_RELDATE
")\n";
40 MODULE_AUTHOR("David S. Miller (davem@davemloft.net)");
41 MODULE_DESCRIPTION("Niagara2 Crypto driver");
42 MODULE_LICENSE("GPL");
43 MODULE_VERSION(DRV_MODULE_VERSION
);
45 #define N2_CRA_PRIORITY 300
47 static DEFINE_MUTEX(spu_lock
);
51 unsigned long qhandle
;
58 struct list_head jobs
;
65 struct list_head list
;
68 static struct spu_queue
**cpu_to_cwq
;
69 static struct spu_queue
**cpu_to_mau
;
71 static unsigned long spu_next_offset(struct spu_queue
*q
, unsigned long off
)
73 if (q
->q_type
== HV_NCS_QTYPE_MAU
) {
74 off
+= MAU_ENTRY_SIZE
;
75 if (off
== (MAU_ENTRY_SIZE
* MAU_NUM_ENTRIES
))
78 off
+= CWQ_ENTRY_SIZE
;
79 if (off
== (CWQ_ENTRY_SIZE
* CWQ_NUM_ENTRIES
))
85 struct n2_request_common
{
86 struct list_head entry
;
89 #define OFFSET_NOT_RUNNING (~(unsigned int)0)
91 /* An async job request records the final tail value it used in
92 * n2_request_common->offset, test to see if that offset is in
93 * the range old_head, new_head, inclusive.
95 static inline bool job_finished(struct spu_queue
*q
, unsigned int offset
,
96 unsigned long old_head
, unsigned long new_head
)
98 if (old_head
<= new_head
) {
99 if (offset
> old_head
&& offset
<= new_head
)
102 if (offset
> old_head
|| offset
<= new_head
)
108 /* When the HEAD marker is unequal to the actual HEAD, we get
109 * a virtual device INO interrupt. We should process the
110 * completed CWQ entries and adjust the HEAD marker to clear
113 static irqreturn_t
cwq_intr(int irq
, void *dev_id
)
115 unsigned long off
, new_head
, hv_ret
;
116 struct spu_queue
*q
= dev_id
;
118 pr_err("CPU[%d]: Got CWQ interrupt for qhdl[%lx]\n",
119 smp_processor_id(), q
->qhandle
);
123 hv_ret
= sun4v_ncs_gethead(q
->qhandle
, &new_head
);
125 pr_err("CPU[%d]: CWQ gethead[%lx] hv_ret[%lu]\n",
126 smp_processor_id(), new_head
, hv_ret
);
128 for (off
= q
->head
; off
!= new_head
; off
= spu_next_offset(q
, off
)) {
132 hv_ret
= sun4v_ncs_sethead_marker(q
->qhandle
, new_head
);
133 if (hv_ret
== HV_EOK
)
136 spin_unlock(&q
->lock
);
141 static irqreturn_t
mau_intr(int irq
, void *dev_id
)
143 struct spu_queue
*q
= dev_id
;
144 unsigned long head
, hv_ret
;
148 pr_err("CPU[%d]: Got MAU interrupt for qhdl[%lx]\n",
149 smp_processor_id(), q
->qhandle
);
151 hv_ret
= sun4v_ncs_gethead(q
->qhandle
, &head
);
153 pr_err("CPU[%d]: MAU gethead[%lx] hv_ret[%lu]\n",
154 smp_processor_id(), head
, hv_ret
);
156 sun4v_ncs_sethead_marker(q
->qhandle
, head
);
158 spin_unlock(&q
->lock
);
163 static void *spu_queue_next(struct spu_queue
*q
, void *cur
)
165 return q
->q
+ spu_next_offset(q
, cur
- q
->q
);
168 static int spu_queue_num_free(struct spu_queue
*q
)
170 unsigned long head
= q
->head
;
171 unsigned long tail
= q
->tail
;
172 unsigned long end
= (CWQ_ENTRY_SIZE
* CWQ_NUM_ENTRIES
);
178 diff
= (end
- tail
) + head
;
180 return (diff
/ CWQ_ENTRY_SIZE
) - 1;
183 static void *spu_queue_alloc(struct spu_queue
*q
, int num_entries
)
185 int avail
= spu_queue_num_free(q
);
187 if (avail
>= num_entries
)
188 return q
->q
+ q
->tail
;
193 static unsigned long spu_queue_submit(struct spu_queue
*q
, void *last
)
195 unsigned long hv_ret
, new_tail
;
197 new_tail
= spu_next_offset(q
, last
- q
->q
);
199 hv_ret
= sun4v_ncs_settail(q
->qhandle
, new_tail
);
200 if (hv_ret
== HV_EOK
)
205 static u64
control_word_base(unsigned int len
, unsigned int hmac_key_len
,
206 int enc_type
, int auth_type
,
207 unsigned int hash_len
,
208 bool sfas
, bool sob
, bool eob
, bool encrypt
,
211 u64 word
= (len
- 1) & CONTROL_LEN
;
213 word
|= ((u64
) opcode
<< CONTROL_OPCODE_SHIFT
);
214 word
|= ((u64
) enc_type
<< CONTROL_ENC_TYPE_SHIFT
);
215 word
|= ((u64
) auth_type
<< CONTROL_AUTH_TYPE_SHIFT
);
217 word
|= CONTROL_STORE_FINAL_AUTH_STATE
;
219 word
|= CONTROL_START_OF_BLOCK
;
221 word
|= CONTROL_END_OF_BLOCK
;
223 word
|= CONTROL_ENCRYPT
;
225 word
|= ((u64
) (hmac_key_len
- 1)) << CONTROL_HMAC_KEY_LEN_SHIFT
;
227 word
|= ((u64
) (hash_len
- 1)) << CONTROL_HASH_LEN_SHIFT
;
233 static inline bool n2_should_run_async(struct spu_queue
*qp
, int this_len
)
235 if (this_len
>= 64 ||
236 qp
->head
!= qp
->tail
)
243 struct crypto_ahash
*fallback_tfm
;
246 struct n2_hash_req_ctx
{
248 struct md5_state md5
;
249 struct sha1_state sha1
;
250 struct sha256_state sha256
;
253 unsigned char hash_key
[64];
254 unsigned char keyed_zero_hash
[32];
256 struct ahash_request fallback_req
;
259 static int n2_hash_async_init(struct ahash_request
*req
)
261 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
262 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
263 struct n2_hash_ctx
*ctx
= crypto_ahash_ctx(tfm
);
265 ahash_request_set_tfm(&rctx
->fallback_req
, ctx
->fallback_tfm
);
266 rctx
->fallback_req
.base
.flags
= req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
;
268 return crypto_ahash_init(&rctx
->fallback_req
);
271 static int n2_hash_async_update(struct ahash_request
*req
)
273 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
274 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
275 struct n2_hash_ctx
*ctx
= crypto_ahash_ctx(tfm
);
277 ahash_request_set_tfm(&rctx
->fallback_req
, ctx
->fallback_tfm
);
278 rctx
->fallback_req
.base
.flags
= req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
;
279 rctx
->fallback_req
.nbytes
= req
->nbytes
;
280 rctx
->fallback_req
.src
= req
->src
;
282 return crypto_ahash_update(&rctx
->fallback_req
);
285 static int n2_hash_async_final(struct ahash_request
*req
)
287 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
288 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
289 struct n2_hash_ctx
*ctx
= crypto_ahash_ctx(tfm
);
291 ahash_request_set_tfm(&rctx
->fallback_req
, ctx
->fallback_tfm
);
292 rctx
->fallback_req
.base
.flags
= req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
;
293 rctx
->fallback_req
.result
= req
->result
;
295 return crypto_ahash_final(&rctx
->fallback_req
);
298 static int n2_hash_async_finup(struct ahash_request
*req
)
300 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
301 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
302 struct n2_hash_ctx
*ctx
= crypto_ahash_ctx(tfm
);
304 ahash_request_set_tfm(&rctx
->fallback_req
, ctx
->fallback_tfm
);
305 rctx
->fallback_req
.base
.flags
= req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
;
306 rctx
->fallback_req
.nbytes
= req
->nbytes
;
307 rctx
->fallback_req
.src
= req
->src
;
308 rctx
->fallback_req
.result
= req
->result
;
310 return crypto_ahash_finup(&rctx
->fallback_req
);
313 static int n2_hash_cra_init(struct crypto_tfm
*tfm
)
315 const char *fallback_driver_name
= tfm
->__crt_alg
->cra_name
;
316 struct crypto_ahash
*ahash
= __crypto_ahash_cast(tfm
);
317 struct n2_hash_ctx
*ctx
= crypto_ahash_ctx(ahash
);
318 struct crypto_ahash
*fallback_tfm
;
321 fallback_tfm
= crypto_alloc_ahash(fallback_driver_name
, 0,
322 CRYPTO_ALG_NEED_FALLBACK
);
323 if (IS_ERR(fallback_tfm
)) {
324 pr_warning("Fallback driver '%s' could not be loaded!\n",
325 fallback_driver_name
);
326 err
= PTR_ERR(fallback_tfm
);
330 crypto_ahash_set_reqsize(ahash
, (sizeof(struct n2_hash_req_ctx
) +
331 crypto_ahash_reqsize(fallback_tfm
)));
333 ctx
->fallback_tfm
= fallback_tfm
;
340 static void n2_hash_cra_exit(struct crypto_tfm
*tfm
)
342 struct crypto_ahash
*ahash
= __crypto_ahash_cast(tfm
);
343 struct n2_hash_ctx
*ctx
= crypto_ahash_ctx(ahash
);
345 crypto_free_ahash(ctx
->fallback_tfm
);
348 static unsigned long wait_for_tail(struct spu_queue
*qp
)
350 unsigned long head
, hv_ret
;
353 hv_ret
= sun4v_ncs_gethead(qp
->qhandle
, &head
);
354 if (hv_ret
!= HV_EOK
) {
355 pr_err("Hypervisor error on gethead\n");
358 if (head
== qp
->tail
) {
366 static unsigned long submit_and_wait_for_tail(struct spu_queue
*qp
,
367 struct cwq_initial_entry
*ent
)
369 unsigned long hv_ret
= spu_queue_submit(qp
, ent
);
371 if (hv_ret
== HV_EOK
)
372 hv_ret
= wait_for_tail(qp
);
377 static int n2_hash_async_digest(struct ahash_request
*req
,
378 unsigned int auth_type
, unsigned int digest_size
,
379 unsigned int result_size
, void *hash_loc
)
381 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
382 struct cwq_initial_entry
*ent
;
383 struct crypto_hash_walk walk
;
384 struct spu_queue
*qp
;
389 /* The total effective length of the operation may not
392 if (unlikely(req
->nbytes
> (1 << 16))) {
393 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
394 struct n2_hash_ctx
*ctx
= crypto_ahash_ctx(tfm
);
396 ahash_request_set_tfm(&rctx
->fallback_req
, ctx
->fallback_tfm
);
397 rctx
->fallback_req
.base
.flags
=
398 req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
;
399 rctx
->fallback_req
.nbytes
= req
->nbytes
;
400 rctx
->fallback_req
.src
= req
->src
;
401 rctx
->fallback_req
.result
= req
->result
;
403 return crypto_ahash_digest(&rctx
->fallback_req
);
406 nbytes
= crypto_hash_walk_first(req
, &walk
);
409 qp
= cpu_to_cwq
[cpu
];
413 spin_lock_irqsave(&qp
->lock
, flags
);
415 /* XXX can do better, improve this later by doing a by-hand scatterlist
418 ent
= qp
->q
+ qp
->tail
;
420 ent
->control
= control_word_base(nbytes
, 0, 0,
421 auth_type
, digest_size
,
422 false, true, false, false,
425 ent
->src_addr
= __pa(walk
.data
);
426 ent
->auth_key_addr
= 0UL;
427 ent
->auth_iv_addr
= __pa(hash_loc
);
428 ent
->final_auth_state_addr
= 0UL;
429 ent
->enc_key_addr
= 0UL;
430 ent
->enc_iv_addr
= 0UL;
431 ent
->dest_addr
= __pa(hash_loc
);
433 nbytes
= crypto_hash_walk_done(&walk
, 0);
435 ent
= spu_queue_next(qp
, ent
);
437 ent
->control
= (nbytes
- 1);
438 ent
->src_addr
= __pa(walk
.data
);
439 ent
->auth_key_addr
= 0UL;
440 ent
->auth_iv_addr
= 0UL;
441 ent
->final_auth_state_addr
= 0UL;
442 ent
->enc_key_addr
= 0UL;
443 ent
->enc_iv_addr
= 0UL;
444 ent
->dest_addr
= 0UL;
446 nbytes
= crypto_hash_walk_done(&walk
, 0);
448 ent
->control
|= CONTROL_END_OF_BLOCK
;
450 if (submit_and_wait_for_tail(qp
, ent
) != HV_EOK
)
455 spin_unlock_irqrestore(&qp
->lock
, flags
);
458 memcpy(req
->result
, hash_loc
, result_size
);
465 static int n2_md5_async_digest(struct ahash_request
*req
)
467 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
468 struct md5_state
*m
= &rctx
->u
.md5
;
470 if (unlikely(req
->nbytes
== 0)) {
471 static const char md5_zero
[MD5_DIGEST_SIZE
] = {
472 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04,
473 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e,
476 memcpy(req
->result
, md5_zero
, MD5_DIGEST_SIZE
);
479 m
->hash
[0] = cpu_to_le32(0x67452301);
480 m
->hash
[1] = cpu_to_le32(0xefcdab89);
481 m
->hash
[2] = cpu_to_le32(0x98badcfe);
482 m
->hash
[3] = cpu_to_le32(0x10325476);
484 return n2_hash_async_digest(req
, AUTH_TYPE_MD5
,
485 MD5_DIGEST_SIZE
, MD5_DIGEST_SIZE
,
489 static int n2_sha1_async_digest(struct ahash_request
*req
)
491 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
492 struct sha1_state
*s
= &rctx
->u
.sha1
;
494 if (unlikely(req
->nbytes
== 0)) {
495 static const char sha1_zero
[SHA1_DIGEST_SIZE
] = {
496 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32,
497 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8,
501 memcpy(req
->result
, sha1_zero
, SHA1_DIGEST_SIZE
);
504 s
->state
[0] = SHA1_H0
;
505 s
->state
[1] = SHA1_H1
;
506 s
->state
[2] = SHA1_H2
;
507 s
->state
[3] = SHA1_H3
;
508 s
->state
[4] = SHA1_H4
;
510 return n2_hash_async_digest(req
, AUTH_TYPE_SHA1
,
511 SHA1_DIGEST_SIZE
, SHA1_DIGEST_SIZE
,
515 static int n2_sha256_async_digest(struct ahash_request
*req
)
517 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
518 struct sha256_state
*s
= &rctx
->u
.sha256
;
520 if (req
->nbytes
== 0) {
521 static const char sha256_zero
[SHA256_DIGEST_SIZE
] = {
522 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a,
523 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae,
524 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99,
525 0x1b, 0x78, 0x52, 0xb8, 0x55
528 memcpy(req
->result
, sha256_zero
, SHA256_DIGEST_SIZE
);
531 s
->state
[0] = SHA256_H0
;
532 s
->state
[1] = SHA256_H1
;
533 s
->state
[2] = SHA256_H2
;
534 s
->state
[3] = SHA256_H3
;
535 s
->state
[4] = SHA256_H4
;
536 s
->state
[5] = SHA256_H5
;
537 s
->state
[6] = SHA256_H6
;
538 s
->state
[7] = SHA256_H7
;
540 return n2_hash_async_digest(req
, AUTH_TYPE_SHA256
,
541 SHA256_DIGEST_SIZE
, SHA256_DIGEST_SIZE
,
545 static int n2_sha224_async_digest(struct ahash_request
*req
)
547 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
548 struct sha256_state
*s
= &rctx
->u
.sha256
;
550 if (req
->nbytes
== 0) {
551 static const char sha224_zero
[SHA224_DIGEST_SIZE
] = {
552 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47,
553 0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2,
554 0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 0xc5, 0xb3, 0xe4,
558 memcpy(req
->result
, sha224_zero
, SHA224_DIGEST_SIZE
);
561 s
->state
[0] = SHA224_H0
;
562 s
->state
[1] = SHA224_H1
;
563 s
->state
[2] = SHA224_H2
;
564 s
->state
[3] = SHA224_H3
;
565 s
->state
[4] = SHA224_H4
;
566 s
->state
[5] = SHA224_H5
;
567 s
->state
[6] = SHA224_H6
;
568 s
->state
[7] = SHA224_H7
;
570 return n2_hash_async_digest(req
, AUTH_TYPE_SHA256
,
571 SHA256_DIGEST_SIZE
, SHA224_DIGEST_SIZE
,
575 struct n2_cipher_context
{
579 u8 aes
[AES_MAX_KEY_SIZE
];
580 u8 des
[DES_KEY_SIZE
];
581 u8 des3
[3 * DES_KEY_SIZE
];
582 u8 arc4
[258]; /* S-box, X, Y */
586 #define N2_CHUNK_ARR_LEN 16
588 struct n2_crypto_chunk
{
589 struct list_head entry
;
590 unsigned long iv_paddr
: 44;
591 unsigned long arr_len
: 20;
592 unsigned long dest_paddr
;
593 unsigned long dest_final
;
595 unsigned long src_paddr
: 44;
596 unsigned long src_len
: 20;
597 } arr
[N2_CHUNK_ARR_LEN
];
600 struct n2_request_context
{
601 struct ablkcipher_walk walk
;
602 struct list_head chunk_list
;
603 struct n2_crypto_chunk chunk
;
607 /* The SPU allows some level of flexibility for partial cipher blocks
608 * being specified in a descriptor.
610 * It merely requires that every descriptor's length field is at least
611 * as large as the cipher block size. This means that a cipher block
612 * can span at most 2 descriptors. However, this does not allow a
613 * partial block to span into the final descriptor as that would
614 * violate the rule (since every descriptor's length must be at lest
615 * the block size). So, for example, assuming an 8 byte block size:
617 * 0xe --> 0xa --> 0x8
619 * is a valid length sequence, whereas:
621 * 0xe --> 0xb --> 0x7
623 * is not a valid sequence.
626 struct n2_cipher_alg
{
627 struct list_head entry
;
629 struct crypto_alg alg
;
632 static inline struct n2_cipher_alg
*n2_cipher_alg(struct crypto_tfm
*tfm
)
634 struct crypto_alg
*alg
= tfm
->__crt_alg
;
636 return container_of(alg
, struct n2_cipher_alg
, alg
);
639 struct n2_cipher_request_context
{
640 struct ablkcipher_walk walk
;
643 static int n2_aes_setkey(struct crypto_ablkcipher
*cipher
, const u8
*key
,
646 struct crypto_tfm
*tfm
= crypto_ablkcipher_tfm(cipher
);
647 struct n2_cipher_context
*ctx
= crypto_tfm_ctx(tfm
);
648 struct n2_cipher_alg
*n2alg
= n2_cipher_alg(tfm
);
650 ctx
->enc_type
= (n2alg
->enc_type
& ENC_TYPE_CHAINING_MASK
);
653 case AES_KEYSIZE_128
:
654 ctx
->enc_type
|= ENC_TYPE_ALG_AES128
;
656 case AES_KEYSIZE_192
:
657 ctx
->enc_type
|= ENC_TYPE_ALG_AES192
;
659 case AES_KEYSIZE_256
:
660 ctx
->enc_type
|= ENC_TYPE_ALG_AES256
;
663 crypto_ablkcipher_set_flags(cipher
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
667 ctx
->key_len
= keylen
;
668 memcpy(ctx
->key
.aes
, key
, keylen
);
672 static int n2_des_setkey(struct crypto_ablkcipher
*cipher
, const u8
*key
,
675 struct crypto_tfm
*tfm
= crypto_ablkcipher_tfm(cipher
);
676 struct n2_cipher_context
*ctx
= crypto_tfm_ctx(tfm
);
677 struct n2_cipher_alg
*n2alg
= n2_cipher_alg(tfm
);
678 u32 tmp
[DES_EXPKEY_WORDS
];
681 ctx
->enc_type
= n2alg
->enc_type
;
683 if (keylen
!= DES_KEY_SIZE
) {
684 crypto_ablkcipher_set_flags(cipher
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
688 err
= des_ekey(tmp
, key
);
689 if (err
== 0 && (tfm
->crt_flags
& CRYPTO_TFM_REQ_WEAK_KEY
)) {
690 tfm
->crt_flags
|= CRYPTO_TFM_RES_WEAK_KEY
;
694 ctx
->key_len
= keylen
;
695 memcpy(ctx
->key
.des
, key
, keylen
);
699 static int n2_3des_setkey(struct crypto_ablkcipher
*cipher
, const u8
*key
,
702 struct crypto_tfm
*tfm
= crypto_ablkcipher_tfm(cipher
);
703 struct n2_cipher_context
*ctx
= crypto_tfm_ctx(tfm
);
704 struct n2_cipher_alg
*n2alg
= n2_cipher_alg(tfm
);
706 ctx
->enc_type
= n2alg
->enc_type
;
708 if (keylen
!= (3 * DES_KEY_SIZE
)) {
709 crypto_ablkcipher_set_flags(cipher
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
712 ctx
->key_len
= keylen
;
713 memcpy(ctx
->key
.des3
, key
, keylen
);
717 static int n2_arc4_setkey(struct crypto_ablkcipher
*cipher
, const u8
*key
,
720 struct crypto_tfm
*tfm
= crypto_ablkcipher_tfm(cipher
);
721 struct n2_cipher_context
*ctx
= crypto_tfm_ctx(tfm
);
722 struct n2_cipher_alg
*n2alg
= n2_cipher_alg(tfm
);
723 u8
*s
= ctx
->key
.arc4
;
728 ctx
->enc_type
= n2alg
->enc_type
;
733 for (i
= 0; i
< 256; i
++)
735 for (i
= 0; i
< 256; i
++) {
737 j
= (j
+ key
[k
] + a
) & 0xff;
747 static inline int cipher_descriptor_len(int nbytes
, unsigned int block_size
)
749 int this_len
= nbytes
;
751 this_len
-= (nbytes
& (block_size
- 1));
752 return this_len
> (1 << 16) ? (1 << 16) : this_len
;
755 static int __n2_crypt_chunk(struct crypto_tfm
*tfm
, struct n2_crypto_chunk
*cp
,
756 struct spu_queue
*qp
, bool encrypt
)
758 struct n2_cipher_context
*ctx
= crypto_tfm_ctx(tfm
);
759 struct cwq_initial_entry
*ent
;
763 ent
= spu_queue_alloc(qp
, cp
->arr_len
);
765 pr_info("queue_alloc() of %d fails\n",
770 in_place
= (cp
->dest_paddr
== cp
->arr
[0].src_paddr
);
772 ent
->control
= control_word_base(cp
->arr
[0].src_len
,
773 0, ctx
->enc_type
, 0, 0,
774 false, true, false, encrypt
,
776 (in_place
? OPCODE_INPLACE_BIT
: 0));
777 ent
->src_addr
= cp
->arr
[0].src_paddr
;
778 ent
->auth_key_addr
= 0UL;
779 ent
->auth_iv_addr
= 0UL;
780 ent
->final_auth_state_addr
= 0UL;
781 ent
->enc_key_addr
= __pa(&ctx
->key
);
782 ent
->enc_iv_addr
= cp
->iv_paddr
;
783 ent
->dest_addr
= (in_place
? 0UL : cp
->dest_paddr
);
785 for (i
= 1; i
< cp
->arr_len
; i
++) {
786 ent
= spu_queue_next(qp
, ent
);
788 ent
->control
= cp
->arr
[i
].src_len
- 1;
789 ent
->src_addr
= cp
->arr
[i
].src_paddr
;
790 ent
->auth_key_addr
= 0UL;
791 ent
->auth_iv_addr
= 0UL;
792 ent
->final_auth_state_addr
= 0UL;
793 ent
->enc_key_addr
= 0UL;
794 ent
->enc_iv_addr
= 0UL;
795 ent
->dest_addr
= 0UL;
797 ent
->control
|= CONTROL_END_OF_BLOCK
;
799 return (spu_queue_submit(qp
, ent
) != HV_EOK
) ? -EINVAL
: 0;
802 static int n2_compute_chunks(struct ablkcipher_request
*req
)
804 struct n2_request_context
*rctx
= ablkcipher_request_ctx(req
);
805 struct ablkcipher_walk
*walk
= &rctx
->walk
;
806 struct n2_crypto_chunk
*chunk
;
807 unsigned long dest_prev
;
808 unsigned int tot_len
;
812 ablkcipher_walk_init(walk
, req
->dst
, req
->src
, req
->nbytes
);
813 err
= ablkcipher_walk_phys(req
, walk
);
817 INIT_LIST_HEAD(&rctx
->chunk_list
);
819 chunk
= &rctx
->chunk
;
820 INIT_LIST_HEAD(&chunk
->entry
);
822 chunk
->iv_paddr
= 0UL;
824 chunk
->dest_paddr
= 0UL;
826 prev_in_place
= false;
830 while ((nbytes
= walk
->nbytes
) != 0) {
831 unsigned long dest_paddr
, src_paddr
;
835 src_paddr
= (page_to_phys(walk
->src
.page
) +
837 dest_paddr
= (page_to_phys(walk
->dst
.page
) +
839 in_place
= (src_paddr
== dest_paddr
);
840 this_len
= cipher_descriptor_len(nbytes
, walk
->blocksize
);
842 if (chunk
->arr_len
!= 0) {
843 if (in_place
!= prev_in_place
||
845 dest_paddr
!= dest_prev
) ||
846 chunk
->arr_len
== N2_CHUNK_ARR_LEN
||
847 tot_len
+ this_len
> (1 << 16)) {
848 chunk
->dest_final
= dest_prev
;
849 list_add_tail(&chunk
->entry
,
851 chunk
= kzalloc(sizeof(*chunk
), GFP_ATOMIC
);
856 INIT_LIST_HEAD(&chunk
->entry
);
859 if (chunk
->arr_len
== 0) {
860 chunk
->dest_paddr
= dest_paddr
;
863 chunk
->arr
[chunk
->arr_len
].src_paddr
= src_paddr
;
864 chunk
->arr
[chunk
->arr_len
].src_len
= this_len
;
867 dest_prev
= dest_paddr
+ this_len
;
868 prev_in_place
= in_place
;
871 err
= ablkcipher_walk_done(req
, walk
, nbytes
- this_len
);
875 if (!err
&& chunk
->arr_len
!= 0) {
876 chunk
->dest_final
= dest_prev
;
877 list_add_tail(&chunk
->entry
, &rctx
->chunk_list
);
883 static void n2_chunk_complete(struct ablkcipher_request
*req
, void *final_iv
)
885 struct n2_request_context
*rctx
= ablkcipher_request_ctx(req
);
886 struct n2_crypto_chunk
*c
, *tmp
;
889 memcpy(rctx
->walk
.iv
, final_iv
, rctx
->walk
.blocksize
);
891 ablkcipher_walk_complete(&rctx
->walk
);
892 list_for_each_entry_safe(c
, tmp
, &rctx
->chunk_list
, entry
) {
894 if (unlikely(c
!= &rctx
->chunk
))
900 static int n2_do_ecb(struct ablkcipher_request
*req
, bool encrypt
)
902 struct n2_request_context
*rctx
= ablkcipher_request_ctx(req
);
903 struct crypto_tfm
*tfm
= req
->base
.tfm
;
904 int err
= n2_compute_chunks(req
);
905 struct n2_crypto_chunk
*c
, *tmp
;
906 unsigned long flags
, hv_ret
;
907 struct spu_queue
*qp
;
912 qp
= cpu_to_cwq
[get_cpu()];
917 spin_lock_irqsave(&qp
->lock
, flags
);
919 list_for_each_entry_safe(c
, tmp
, &rctx
->chunk_list
, entry
) {
920 err
= __n2_crypt_chunk(tfm
, c
, qp
, encrypt
);
924 if (unlikely(c
!= &rctx
->chunk
))
928 hv_ret
= wait_for_tail(qp
);
929 if (hv_ret
!= HV_EOK
)
933 spin_unlock_irqrestore(&qp
->lock
, flags
);
938 n2_chunk_complete(req
, NULL
);
942 static int n2_encrypt_ecb(struct ablkcipher_request
*req
)
944 return n2_do_ecb(req
, true);
947 static int n2_decrypt_ecb(struct ablkcipher_request
*req
)
949 return n2_do_ecb(req
, false);
952 static int n2_do_chaining(struct ablkcipher_request
*req
, bool encrypt
)
954 struct n2_request_context
*rctx
= ablkcipher_request_ctx(req
);
955 struct crypto_tfm
*tfm
= req
->base
.tfm
;
956 unsigned long flags
, hv_ret
, iv_paddr
;
957 int err
= n2_compute_chunks(req
);
958 struct n2_crypto_chunk
*c
, *tmp
;
959 struct spu_queue
*qp
;
962 final_iv_addr
= NULL
;
967 qp
= cpu_to_cwq
[get_cpu()];
972 spin_lock_irqsave(&qp
->lock
, flags
);
975 iv_paddr
= __pa(rctx
->walk
.iv
);
976 list_for_each_entry_safe(c
, tmp
, &rctx
->chunk_list
,
978 c
->iv_paddr
= iv_paddr
;
979 err
= __n2_crypt_chunk(tfm
, c
, qp
, true);
982 iv_paddr
= c
->dest_final
- rctx
->walk
.blocksize
;
984 if (unlikely(c
!= &rctx
->chunk
))
987 final_iv_addr
= __va(iv_paddr
);
989 list_for_each_entry_safe_reverse(c
, tmp
, &rctx
->chunk_list
,
991 if (c
== &rctx
->chunk
) {
992 iv_paddr
= __pa(rctx
->walk
.iv
);
994 iv_paddr
= (tmp
->arr
[tmp
->arr_len
-1].src_paddr
+
995 tmp
->arr
[tmp
->arr_len
-1].src_len
-
996 rctx
->walk
.blocksize
);
998 if (!final_iv_addr
) {
1001 pa
= (c
->arr
[c
->arr_len
-1].src_paddr
+
1002 c
->arr
[c
->arr_len
-1].src_len
-
1003 rctx
->walk
.blocksize
);
1004 final_iv_addr
= rctx
->temp_iv
;
1005 memcpy(rctx
->temp_iv
, __va(pa
),
1006 rctx
->walk
.blocksize
);
1008 c
->iv_paddr
= iv_paddr
;
1009 err
= __n2_crypt_chunk(tfm
, c
, qp
, false);
1012 list_del(&c
->entry
);
1013 if (unlikely(c
!= &rctx
->chunk
))
1018 hv_ret
= wait_for_tail(qp
);
1019 if (hv_ret
!= HV_EOK
)
1023 spin_unlock_irqrestore(&qp
->lock
, flags
);
1028 n2_chunk_complete(req
, err
? NULL
: final_iv_addr
);
1032 static int n2_encrypt_chaining(struct ablkcipher_request
*req
)
1034 return n2_do_chaining(req
, true);
1037 static int n2_decrypt_chaining(struct ablkcipher_request
*req
)
1039 return n2_do_chaining(req
, false);
1042 struct n2_cipher_tmpl
{
1044 const char *drv_name
;
1047 struct ablkcipher_alg ablkcipher
;
1050 static const struct n2_cipher_tmpl cipher_tmpls
[] = {
1051 /* ARC4: only ECB is supported (chaining bits ignored) */
1052 { .name
= "ecb(arc4)",
1053 .drv_name
= "ecb-arc4",
1055 .enc_type
= (ENC_TYPE_ALG_RC4_STREAM
|
1056 ENC_TYPE_CHAINING_ECB
),
1060 .setkey
= n2_arc4_setkey
,
1061 .encrypt
= n2_encrypt_ecb
,
1062 .decrypt
= n2_decrypt_ecb
,
1066 /* DES: ECB CBC and CFB are supported */
1067 { .name
= "ecb(des)",
1068 .drv_name
= "ecb-des",
1069 .block_size
= DES_BLOCK_SIZE
,
1070 .enc_type
= (ENC_TYPE_ALG_DES
|
1071 ENC_TYPE_CHAINING_ECB
),
1073 .min_keysize
= DES_KEY_SIZE
,
1074 .max_keysize
= DES_KEY_SIZE
,
1075 .setkey
= n2_des_setkey
,
1076 .encrypt
= n2_encrypt_ecb
,
1077 .decrypt
= n2_decrypt_ecb
,
1080 { .name
= "cbc(des)",
1081 .drv_name
= "cbc-des",
1082 .block_size
= DES_BLOCK_SIZE
,
1083 .enc_type
= (ENC_TYPE_ALG_DES
|
1084 ENC_TYPE_CHAINING_CBC
),
1086 .ivsize
= DES_BLOCK_SIZE
,
1087 .min_keysize
= DES_KEY_SIZE
,
1088 .max_keysize
= DES_KEY_SIZE
,
1089 .setkey
= n2_des_setkey
,
1090 .encrypt
= n2_encrypt_chaining
,
1091 .decrypt
= n2_decrypt_chaining
,
1094 { .name
= "cfb(des)",
1095 .drv_name
= "cfb-des",
1096 .block_size
= DES_BLOCK_SIZE
,
1097 .enc_type
= (ENC_TYPE_ALG_DES
|
1098 ENC_TYPE_CHAINING_CFB
),
1100 .min_keysize
= DES_KEY_SIZE
,
1101 .max_keysize
= DES_KEY_SIZE
,
1102 .setkey
= n2_des_setkey
,
1103 .encrypt
= n2_encrypt_chaining
,
1104 .decrypt
= n2_decrypt_chaining
,
1108 /* 3DES: ECB CBC and CFB are supported */
1109 { .name
= "ecb(des3_ede)",
1110 .drv_name
= "ecb-3des",
1111 .block_size
= DES_BLOCK_SIZE
,
1112 .enc_type
= (ENC_TYPE_ALG_3DES
|
1113 ENC_TYPE_CHAINING_ECB
),
1115 .min_keysize
= 3 * DES_KEY_SIZE
,
1116 .max_keysize
= 3 * DES_KEY_SIZE
,
1117 .setkey
= n2_3des_setkey
,
1118 .encrypt
= n2_encrypt_ecb
,
1119 .decrypt
= n2_decrypt_ecb
,
1122 { .name
= "cbc(des3_ede)",
1123 .drv_name
= "cbc-3des",
1124 .block_size
= DES_BLOCK_SIZE
,
1125 .enc_type
= (ENC_TYPE_ALG_3DES
|
1126 ENC_TYPE_CHAINING_CBC
),
1128 .ivsize
= DES_BLOCK_SIZE
,
1129 .min_keysize
= 3 * DES_KEY_SIZE
,
1130 .max_keysize
= 3 * DES_KEY_SIZE
,
1131 .setkey
= n2_3des_setkey
,
1132 .encrypt
= n2_encrypt_chaining
,
1133 .decrypt
= n2_decrypt_chaining
,
1136 { .name
= "cfb(des3_ede)",
1137 .drv_name
= "cfb-3des",
1138 .block_size
= DES_BLOCK_SIZE
,
1139 .enc_type
= (ENC_TYPE_ALG_3DES
|
1140 ENC_TYPE_CHAINING_CFB
),
1142 .min_keysize
= 3 * DES_KEY_SIZE
,
1143 .max_keysize
= 3 * DES_KEY_SIZE
,
1144 .setkey
= n2_3des_setkey
,
1145 .encrypt
= n2_encrypt_chaining
,
1146 .decrypt
= n2_decrypt_chaining
,
1149 /* AES: ECB CBC and CTR are supported */
1150 { .name
= "ecb(aes)",
1151 .drv_name
= "ecb-aes",
1152 .block_size
= AES_BLOCK_SIZE
,
1153 .enc_type
= (ENC_TYPE_ALG_AES128
|
1154 ENC_TYPE_CHAINING_ECB
),
1156 .min_keysize
= AES_MIN_KEY_SIZE
,
1157 .max_keysize
= AES_MAX_KEY_SIZE
,
1158 .setkey
= n2_aes_setkey
,
1159 .encrypt
= n2_encrypt_ecb
,
1160 .decrypt
= n2_decrypt_ecb
,
1163 { .name
= "cbc(aes)",
1164 .drv_name
= "cbc-aes",
1165 .block_size
= AES_BLOCK_SIZE
,
1166 .enc_type
= (ENC_TYPE_ALG_AES128
|
1167 ENC_TYPE_CHAINING_CBC
),
1169 .ivsize
= AES_BLOCK_SIZE
,
1170 .min_keysize
= AES_MIN_KEY_SIZE
,
1171 .max_keysize
= AES_MAX_KEY_SIZE
,
1172 .setkey
= n2_aes_setkey
,
1173 .encrypt
= n2_encrypt_chaining
,
1174 .decrypt
= n2_decrypt_chaining
,
1177 { .name
= "ctr(aes)",
1178 .drv_name
= "ctr-aes",
1179 .block_size
= AES_BLOCK_SIZE
,
1180 .enc_type
= (ENC_TYPE_ALG_AES128
|
1181 ENC_TYPE_CHAINING_COUNTER
),
1183 .ivsize
= AES_BLOCK_SIZE
,
1184 .min_keysize
= AES_MIN_KEY_SIZE
,
1185 .max_keysize
= AES_MAX_KEY_SIZE
,
1186 .setkey
= n2_aes_setkey
,
1187 .encrypt
= n2_encrypt_chaining
,
1188 .decrypt
= n2_encrypt_chaining
,
1193 #define NUM_CIPHER_TMPLS ARRAY_SIZE(cipher_tmpls)
1195 static LIST_HEAD(cipher_algs
);
1197 struct n2_hash_tmpl
{
1199 int (*digest
)(struct ahash_request
*req
);
1203 static const struct n2_hash_tmpl hash_tmpls
[] = {
1205 .digest
= n2_md5_async_digest
,
1206 .digest_size
= MD5_DIGEST_SIZE
,
1207 .block_size
= MD5_HMAC_BLOCK_SIZE
},
1209 .digest
= n2_sha1_async_digest
,
1210 .digest_size
= SHA1_DIGEST_SIZE
,
1211 .block_size
= SHA1_BLOCK_SIZE
},
1213 .digest
= n2_sha256_async_digest
,
1214 .digest_size
= SHA256_DIGEST_SIZE
,
1215 .block_size
= SHA256_BLOCK_SIZE
},
1217 .digest
= n2_sha224_async_digest
,
1218 .digest_size
= SHA224_DIGEST_SIZE
,
1219 .block_size
= SHA224_BLOCK_SIZE
},
1221 #define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls)
1223 struct n2_ahash_alg
{
1224 struct list_head entry
;
1225 struct ahash_alg alg
;
1227 static LIST_HEAD(ahash_algs
);
1229 static int algs_registered
;
1231 static void __n2_unregister_algs(void)
1233 struct n2_cipher_alg
*cipher
, *cipher_tmp
;
1234 struct n2_ahash_alg
*alg
, *alg_tmp
;
1236 list_for_each_entry_safe(cipher
, cipher_tmp
, &cipher_algs
, entry
) {
1237 crypto_unregister_alg(&cipher
->alg
);
1238 list_del(&cipher
->entry
);
1241 list_for_each_entry_safe(alg
, alg_tmp
, &ahash_algs
, entry
) {
1242 crypto_unregister_ahash(&alg
->alg
);
1243 list_del(&alg
->entry
);
1248 static int n2_cipher_cra_init(struct crypto_tfm
*tfm
)
1250 tfm
->crt_ablkcipher
.reqsize
= sizeof(struct n2_request_context
);
1254 static int __devinit
__n2_register_one_cipher(const struct n2_cipher_tmpl
*tmpl
)
1256 struct n2_cipher_alg
*p
= kzalloc(sizeof(*p
), GFP_KERNEL
);
1257 struct crypto_alg
*alg
;
1265 snprintf(alg
->cra_name
, CRYPTO_MAX_ALG_NAME
, "%s", tmpl
->name
);
1266 snprintf(alg
->cra_driver_name
, CRYPTO_MAX_ALG_NAME
, "%s-n2", tmpl
->drv_name
);
1267 alg
->cra_priority
= N2_CRA_PRIORITY
;
1268 alg
->cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
| CRYPTO_ALG_ASYNC
;
1269 alg
->cra_blocksize
= tmpl
->block_size
;
1270 p
->enc_type
= tmpl
->enc_type
;
1271 alg
->cra_ctxsize
= sizeof(struct n2_cipher_context
);
1272 alg
->cra_type
= &crypto_ablkcipher_type
;
1273 alg
->cra_u
.ablkcipher
= tmpl
->ablkcipher
;
1274 alg
->cra_init
= n2_cipher_cra_init
;
1275 alg
->cra_module
= THIS_MODULE
;
1277 list_add(&p
->entry
, &cipher_algs
);
1278 err
= crypto_register_alg(alg
);
1280 pr_err("%s alg registration failed\n", alg
->cra_name
);
1281 list_del(&p
->entry
);
1284 pr_info("%s alg registered\n", alg
->cra_name
);
1289 static int __devinit
__n2_register_one_ahash(const struct n2_hash_tmpl
*tmpl
)
1291 struct n2_ahash_alg
*p
= kzalloc(sizeof(*p
), GFP_KERNEL
);
1292 struct hash_alg_common
*halg
;
1293 struct crypto_alg
*base
;
1294 struct ahash_alg
*ahash
;
1301 ahash
->init
= n2_hash_async_init
;
1302 ahash
->update
= n2_hash_async_update
;
1303 ahash
->final
= n2_hash_async_final
;
1304 ahash
->finup
= n2_hash_async_finup
;
1305 ahash
->digest
= tmpl
->digest
;
1307 halg
= &ahash
->halg
;
1308 halg
->digestsize
= tmpl
->digest_size
;
1311 snprintf(base
->cra_name
, CRYPTO_MAX_ALG_NAME
, "%s", tmpl
->name
);
1312 snprintf(base
->cra_driver_name
, CRYPTO_MAX_ALG_NAME
, "%s-n2", tmpl
->name
);
1313 base
->cra_priority
= N2_CRA_PRIORITY
;
1314 base
->cra_flags
= CRYPTO_ALG_TYPE_AHASH
| CRYPTO_ALG_NEED_FALLBACK
;
1315 base
->cra_blocksize
= tmpl
->block_size
;
1316 base
->cra_ctxsize
= sizeof(struct n2_hash_ctx
);
1317 base
->cra_module
= THIS_MODULE
;
1318 base
->cra_init
= n2_hash_cra_init
;
1319 base
->cra_exit
= n2_hash_cra_exit
;
1321 list_add(&p
->entry
, &ahash_algs
);
1322 err
= crypto_register_ahash(ahash
);
1324 pr_err("%s alg registration failed\n", base
->cra_name
);
1325 list_del(&p
->entry
);
1328 pr_info("%s alg registered\n", base
->cra_name
);
1333 static int __devinit
n2_register_algs(void)
1337 mutex_lock(&spu_lock
);
1338 if (algs_registered
++)
1341 for (i
= 0; i
< NUM_HASH_TMPLS
; i
++) {
1342 err
= __n2_register_one_ahash(&hash_tmpls
[i
]);
1344 __n2_unregister_algs();
1348 for (i
= 0; i
< NUM_CIPHER_TMPLS
; i
++) {
1349 err
= __n2_register_one_cipher(&cipher_tmpls
[i
]);
1351 __n2_unregister_algs();
1357 mutex_unlock(&spu_lock
);
1361 static void __exit
n2_unregister_algs(void)
1363 mutex_lock(&spu_lock
);
1364 if (!--algs_registered
)
1365 __n2_unregister_algs();
1366 mutex_unlock(&spu_lock
);
1369 /* To map CWQ queues to interrupt sources, the hypervisor API provides
1370 * a devino. This isn't very useful to us because all of the
1371 * interrupts listed in the of_device node have been translated to
1372 * Linux virtual IRQ cookie numbers.
1374 * So we have to back-translate, going through the 'intr' and 'ino'
1375 * property tables of the n2cp MDESC node, matching it with the OF
1376 * 'interrupts' property entries, in order to to figure out which
1377 * devino goes to which already-translated IRQ.
1379 static int find_devino_index(struct of_device
*dev
, struct spu_mdesc_info
*ip
,
1380 unsigned long dev_ino
)
1382 const unsigned int *dev_intrs
;
1386 for (i
= 0; i
< ip
->num_intrs
; i
++) {
1387 if (ip
->ino_table
[i
].ino
== dev_ino
)
1390 if (i
== ip
->num_intrs
)
1393 intr
= ip
->ino_table
[i
].intr
;
1395 dev_intrs
= of_get_property(dev
->dev
.of_node
, "interrupts", NULL
);
1399 for (i
= 0; i
< dev
->num_irqs
; i
++) {
1400 if (dev_intrs
[i
] == intr
)
1407 static int spu_map_ino(struct of_device
*dev
, struct spu_mdesc_info
*ip
,
1408 const char *irq_name
, struct spu_queue
*p
,
1409 irq_handler_t handler
)
1414 herr
= sun4v_ncs_qhandle_to_devino(p
->qhandle
, &p
->devino
);
1418 index
= find_devino_index(dev
, ip
, p
->devino
);
1422 p
->irq
= dev
->irqs
[index
];
1424 sprintf(p
->irq_name
, "%s-%d", irq_name
, index
);
1426 return request_irq(p
->irq
, handler
, IRQF_SAMPLE_RANDOM
,
1430 static struct kmem_cache
*queue_cache
[2];
1432 static void *new_queue(unsigned long q_type
)
1434 return kmem_cache_zalloc(queue_cache
[q_type
- 1], GFP_KERNEL
);
1437 static void free_queue(void *p
, unsigned long q_type
)
1439 return kmem_cache_free(queue_cache
[q_type
- 1], p
);
1442 static int queue_cache_init(void)
1444 if (!queue_cache
[HV_NCS_QTYPE_MAU
- 1])
1445 queue_cache
[HV_NCS_QTYPE_MAU
- 1] =
1446 kmem_cache_create("mau_queue",
1449 MAU_ENTRY_SIZE
, 0, NULL
);
1450 if (!queue_cache
[HV_NCS_QTYPE_MAU
- 1])
1453 if (!queue_cache
[HV_NCS_QTYPE_CWQ
- 1])
1454 queue_cache
[HV_NCS_QTYPE_CWQ
- 1] =
1455 kmem_cache_create("cwq_queue",
1458 CWQ_ENTRY_SIZE
, 0, NULL
);
1459 if (!queue_cache
[HV_NCS_QTYPE_CWQ
- 1]) {
1460 kmem_cache_destroy(queue_cache
[HV_NCS_QTYPE_MAU
- 1]);
1466 static void queue_cache_destroy(void)
1468 kmem_cache_destroy(queue_cache
[HV_NCS_QTYPE_MAU
- 1]);
1469 kmem_cache_destroy(queue_cache
[HV_NCS_QTYPE_CWQ
- 1]);
1472 static int spu_queue_register(struct spu_queue
*p
, unsigned long q_type
)
1474 cpumask_var_t old_allowed
;
1475 unsigned long hv_ret
;
1477 if (cpumask_empty(&p
->sharing
))
1480 if (!alloc_cpumask_var(&old_allowed
, GFP_KERNEL
))
1483 cpumask_copy(old_allowed
, ¤t
->cpus_allowed
);
1485 set_cpus_allowed_ptr(current
, &p
->sharing
);
1487 hv_ret
= sun4v_ncs_qconf(q_type
, __pa(p
->q
),
1488 CWQ_NUM_ENTRIES
, &p
->qhandle
);
1490 sun4v_ncs_sethead_marker(p
->qhandle
, 0);
1492 set_cpus_allowed_ptr(current
, old_allowed
);
1494 free_cpumask_var(old_allowed
);
1496 return (hv_ret
? -EINVAL
: 0);
1499 static int spu_queue_setup(struct spu_queue
*p
)
1503 p
->q
= new_queue(p
->q_type
);
1507 err
= spu_queue_register(p
, p
->q_type
);
1509 free_queue(p
->q
, p
->q_type
);
1516 static void spu_queue_destroy(struct spu_queue
*p
)
1518 unsigned long hv_ret
;
1523 hv_ret
= sun4v_ncs_qconf(p
->q_type
, p
->qhandle
, 0, &p
->qhandle
);
1526 free_queue(p
->q
, p
->q_type
);
1529 static void spu_list_destroy(struct list_head
*list
)
1531 struct spu_queue
*p
, *n
;
1533 list_for_each_entry_safe(p
, n
, list
, list
) {
1536 for (i
= 0; i
< NR_CPUS
; i
++) {
1537 if (cpu_to_cwq
[i
] == p
)
1538 cpu_to_cwq
[i
] = NULL
;
1542 free_irq(p
->irq
, p
);
1545 spu_queue_destroy(p
);
1551 /* Walk the backward arcs of a CWQ 'exec-unit' node,
1552 * gathering cpu membership information.
1554 static int spu_mdesc_walk_arcs(struct mdesc_handle
*mdesc
,
1555 struct of_device
*dev
,
1556 u64 node
, struct spu_queue
*p
,
1557 struct spu_queue
**table
)
1561 mdesc_for_each_arc(arc
, mdesc
, node
, MDESC_ARC_TYPE_BACK
) {
1562 u64 tgt
= mdesc_arc_target(mdesc
, arc
);
1563 const char *name
= mdesc_node_name(mdesc
, tgt
);
1566 if (strcmp(name
, "cpu"))
1568 id
= mdesc_get_property(mdesc
, tgt
, "id", NULL
);
1569 if (table
[*id
] != NULL
) {
1570 dev_err(&dev
->dev
, "%s: SPU cpu slot already set.\n",
1571 dev
->dev
.of_node
->full_name
);
1574 cpu_set(*id
, p
->sharing
);
1580 /* Process an 'exec-unit' MDESC node of type 'cwq'. */
1581 static int handle_exec_unit(struct spu_mdesc_info
*ip
, struct list_head
*list
,
1582 struct of_device
*dev
, struct mdesc_handle
*mdesc
,
1583 u64 node
, const char *iname
, unsigned long q_type
,
1584 irq_handler_t handler
, struct spu_queue
**table
)
1586 struct spu_queue
*p
;
1589 p
= kzalloc(sizeof(struct spu_queue
), GFP_KERNEL
);
1591 dev_err(&dev
->dev
, "%s: Could not allocate SPU queue.\n",
1592 dev
->dev
.of_node
->full_name
);
1596 cpus_clear(p
->sharing
);
1597 spin_lock_init(&p
->lock
);
1599 INIT_LIST_HEAD(&p
->jobs
);
1600 list_add(&p
->list
, list
);
1602 err
= spu_mdesc_walk_arcs(mdesc
, dev
, node
, p
, table
);
1606 err
= spu_queue_setup(p
);
1610 return spu_map_ino(dev
, ip
, iname
, p
, handler
);
1613 static int spu_mdesc_scan(struct mdesc_handle
*mdesc
, struct of_device
*dev
,
1614 struct spu_mdesc_info
*ip
, struct list_head
*list
,
1615 const char *exec_name
, unsigned long q_type
,
1616 irq_handler_t handler
, struct spu_queue
**table
)
1621 mdesc_for_each_node_by_name(mdesc
, node
, "exec-unit") {
1624 type
= mdesc_get_property(mdesc
, node
, "type", NULL
);
1625 if (!type
|| strcmp(type
, exec_name
))
1628 err
= handle_exec_unit(ip
, list
, dev
, mdesc
, node
,
1629 exec_name
, q_type
, handler
, table
);
1631 spu_list_destroy(list
);
1639 static int __devinit
get_irq_props(struct mdesc_handle
*mdesc
, u64 node
,
1640 struct spu_mdesc_info
*ip
)
1642 const u64
*intr
, *ino
;
1643 int intr_len
, ino_len
;
1646 intr
= mdesc_get_property(mdesc
, node
, "intr", &intr_len
);
1650 ino
= mdesc_get_property(mdesc
, node
, "ino", &ino_len
);
1654 if (intr_len
!= ino_len
)
1657 ip
->num_intrs
= intr_len
/ sizeof(u64
);
1658 ip
->ino_table
= kzalloc((sizeof(struct ino_blob
) *
1664 for (i
= 0; i
< ip
->num_intrs
; i
++) {
1665 struct ino_blob
*b
= &ip
->ino_table
[i
];
1673 static int __devinit
grab_mdesc_irq_props(struct mdesc_handle
*mdesc
,
1674 struct of_device
*dev
,
1675 struct spu_mdesc_info
*ip
,
1676 const char *node_name
)
1678 const unsigned int *reg
;
1681 reg
= of_get_property(dev
->dev
.of_node
, "reg", NULL
);
1685 mdesc_for_each_node_by_name(mdesc
, node
, "virtual-device") {
1689 name
= mdesc_get_property(mdesc
, node
, "name", NULL
);
1690 if (!name
|| strcmp(name
, node_name
))
1692 chdl
= mdesc_get_property(mdesc
, node
, "cfg-handle", NULL
);
1693 if (!chdl
|| (*chdl
!= *reg
))
1695 ip
->cfg_handle
= *chdl
;
1696 return get_irq_props(mdesc
, node
, ip
);
1702 static unsigned long n2_spu_hvapi_major
;
1703 static unsigned long n2_spu_hvapi_minor
;
1705 static int __devinit
n2_spu_hvapi_register(void)
1709 n2_spu_hvapi_major
= 2;
1710 n2_spu_hvapi_minor
= 0;
1712 err
= sun4v_hvapi_register(HV_GRP_NCS
,
1714 &n2_spu_hvapi_minor
);
1717 pr_info("Registered NCS HVAPI version %lu.%lu\n",
1719 n2_spu_hvapi_minor
);
1724 static void n2_spu_hvapi_unregister(void)
1726 sun4v_hvapi_unregister(HV_GRP_NCS
);
1729 static int global_ref
;
1731 static int __devinit
grab_global_resources(void)
1735 mutex_lock(&spu_lock
);
1740 err
= n2_spu_hvapi_register();
1744 err
= queue_cache_init();
1746 goto out_hvapi_release
;
1749 cpu_to_cwq
= kzalloc(sizeof(struct spu_queue
*) * NR_CPUS
,
1752 goto out_queue_cache_destroy
;
1754 cpu_to_mau
= kzalloc(sizeof(struct spu_queue
*) * NR_CPUS
,
1757 goto out_free_cwq_table
;
1764 mutex_unlock(&spu_lock
);
1771 out_queue_cache_destroy
:
1772 queue_cache_destroy();
1775 n2_spu_hvapi_unregister();
1779 static void release_global_resources(void)
1781 mutex_lock(&spu_lock
);
1782 if (!--global_ref
) {
1789 queue_cache_destroy();
1790 n2_spu_hvapi_unregister();
1792 mutex_unlock(&spu_lock
);
1795 static struct n2_crypto
* __devinit
alloc_n2cp(void)
1797 struct n2_crypto
*np
= kzalloc(sizeof(struct n2_crypto
), GFP_KERNEL
);
1800 INIT_LIST_HEAD(&np
->cwq_list
);
1805 static void free_n2cp(struct n2_crypto
*np
)
1807 if (np
->cwq_info
.ino_table
) {
1808 kfree(np
->cwq_info
.ino_table
);
1809 np
->cwq_info
.ino_table
= NULL
;
1815 static void __devinit
n2_spu_driver_version(void)
1817 static int n2_spu_version_printed
;
1819 if (n2_spu_version_printed
++ == 0)
1820 pr_info("%s", version
);
1823 static int __devinit
n2_crypto_probe(struct of_device
*dev
,
1824 const struct of_device_id
*match
)
1826 struct mdesc_handle
*mdesc
;
1827 const char *full_name
;
1828 struct n2_crypto
*np
;
1831 n2_spu_driver_version();
1833 full_name
= dev
->dev
.of_node
->full_name
;
1834 pr_info("Found N2CP at %s\n", full_name
);
1838 dev_err(&dev
->dev
, "%s: Unable to allocate n2cp.\n",
1843 err
= grab_global_resources();
1845 dev_err(&dev
->dev
, "%s: Unable to grab "
1846 "global resources.\n", full_name
);
1850 mdesc
= mdesc_grab();
1853 dev_err(&dev
->dev
, "%s: Unable to grab MDESC.\n",
1856 goto out_free_global
;
1858 err
= grab_mdesc_irq_props(mdesc
, dev
, &np
->cwq_info
, "n2cp");
1860 dev_err(&dev
->dev
, "%s: Unable to grab IRQ props.\n",
1862 mdesc_release(mdesc
);
1863 goto out_free_global
;
1866 err
= spu_mdesc_scan(mdesc
, dev
, &np
->cwq_info
, &np
->cwq_list
,
1867 "cwq", HV_NCS_QTYPE_CWQ
, cwq_intr
,
1869 mdesc_release(mdesc
);
1872 dev_err(&dev
->dev
, "%s: CWQ MDESC scan failed.\n",
1874 goto out_free_global
;
1877 err
= n2_register_algs();
1879 dev_err(&dev
->dev
, "%s: Unable to register algorithms.\n",
1881 goto out_free_spu_list
;
1884 dev_set_drvdata(&dev
->dev
, np
);
1889 spu_list_destroy(&np
->cwq_list
);
1892 release_global_resources();
1900 static int __devexit
n2_crypto_remove(struct of_device
*dev
)
1902 struct n2_crypto
*np
= dev_get_drvdata(&dev
->dev
);
1904 n2_unregister_algs();
1906 spu_list_destroy(&np
->cwq_list
);
1908 release_global_resources();
1915 static struct n2_mau
* __devinit
alloc_ncp(void)
1917 struct n2_mau
*mp
= kzalloc(sizeof(struct n2_mau
), GFP_KERNEL
);
1920 INIT_LIST_HEAD(&mp
->mau_list
);
1925 static void free_ncp(struct n2_mau
*mp
)
1927 if (mp
->mau_info
.ino_table
) {
1928 kfree(mp
->mau_info
.ino_table
);
1929 mp
->mau_info
.ino_table
= NULL
;
1935 static int __devinit
n2_mau_probe(struct of_device
*dev
,
1936 const struct of_device_id
*match
)
1938 struct mdesc_handle
*mdesc
;
1939 const char *full_name
;
1943 n2_spu_driver_version();
1945 full_name
= dev
->dev
.of_node
->full_name
;
1946 pr_info("Found NCP at %s\n", full_name
);
1950 dev_err(&dev
->dev
, "%s: Unable to allocate ncp.\n",
1955 err
= grab_global_resources();
1957 dev_err(&dev
->dev
, "%s: Unable to grab "
1958 "global resources.\n", full_name
);
1962 mdesc
= mdesc_grab();
1965 dev_err(&dev
->dev
, "%s: Unable to grab MDESC.\n",
1968 goto out_free_global
;
1971 err
= grab_mdesc_irq_props(mdesc
, dev
, &mp
->mau_info
, "ncp");
1973 dev_err(&dev
->dev
, "%s: Unable to grab IRQ props.\n",
1975 mdesc_release(mdesc
);
1976 goto out_free_global
;
1979 err
= spu_mdesc_scan(mdesc
, dev
, &mp
->mau_info
, &mp
->mau_list
,
1980 "mau", HV_NCS_QTYPE_MAU
, mau_intr
,
1982 mdesc_release(mdesc
);
1985 dev_err(&dev
->dev
, "%s: MAU MDESC scan failed.\n",
1987 goto out_free_global
;
1990 dev_set_drvdata(&dev
->dev
, mp
);
1995 release_global_resources();
2003 static int __devexit
n2_mau_remove(struct of_device
*dev
)
2005 struct n2_mau
*mp
= dev_get_drvdata(&dev
->dev
);
2007 spu_list_destroy(&mp
->mau_list
);
2009 release_global_resources();
2016 static struct of_device_id n2_crypto_match
[] = {
2019 .compatible
= "SUNW,n2-cwq",
2023 .compatible
= "SUNW,vf-cwq",
2028 MODULE_DEVICE_TABLE(of
, n2_crypto_match
);
2030 static struct of_platform_driver n2_crypto_driver
= {
2033 .owner
= THIS_MODULE
,
2034 .of_match_table
= n2_crypto_match
,
2036 .probe
= n2_crypto_probe
,
2037 .remove
= __devexit_p(n2_crypto_remove
),
2040 static struct of_device_id n2_mau_match
[] = {
2043 .compatible
= "SUNW,n2-mau",
2047 .compatible
= "SUNW,vf-mau",
2052 MODULE_DEVICE_TABLE(of
, n2_mau_match
);
2054 static struct of_platform_driver n2_mau_driver
= {
2057 .owner
= THIS_MODULE
,
2058 .of_match_table
= n2_mau_match
,
2060 .probe
= n2_mau_probe
,
2061 .remove
= __devexit_p(n2_mau_remove
),
2064 static int __init
n2_init(void)
2066 int err
= of_register_driver(&n2_crypto_driver
, &of_bus_type
);
2069 err
= of_register_driver(&n2_mau_driver
, &of_bus_type
);
2071 of_unregister_driver(&n2_crypto_driver
);
2076 static void __exit
n2_exit(void)
2078 of_unregister_driver(&n2_mau_driver
);
2079 of_unregister_driver(&n2_crypto_driver
);
2082 module_init(n2_init
);
2083 module_exit(n2_exit
);