drivers/crypto/geode-aes.c

   1  /* Copyright (C) 2004-2006, Advanced Micro Devices, Inc.
   2  *
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License as published by
   5  * the Free Software Foundation; either version 2 of the License, or
   6  * (at your option) any later version.
   7  */
   8
   9 #include <linux/module.h>
  10 #include <linux/kernel.h>
  11 #include <linux/pci.h>
  12 #include <linux/pci_ids.h>
  13 #include <linux/crypto.h>
  14 #include <linux/spinlock.h>
  15 #include <crypto/algapi.h>
  16 #include <crypto/aes.h>
  17
  18 #include <asm/io.h>
  19 #include <asm/delay.h>
  20
  21 #include "geode-aes.h"
  22
  23 /* Static structures */
  24
  25 static void __iomem * _iobase;
  26 static spinlock_t lock;
  27
  28 /* Write a 128 bit field (either a writable key or IV) */
  29 static inline void
  30 _writefield(u32 offset, void *value)
  31 {
  32         int i;
  33         for(i = 0; i < 4; i++)
  34                 iowrite32(((u32 *) value)[i], _iobase + offset + (i * 4));
  35 }
  36
  37 /* Read a 128 bit field (either a writable key or IV) */
  38 static inline void
  39 _readfield(u32 offset, void *value)
  40 {
  41         int i;
  42         for(i = 0; i < 4; i++)
  43                 ((u32 *) value)[i] = ioread32(_iobase + offset + (i * 4));
  44 }
  45
  46 static int
  47 do_crypt(void *src, void *dst, int len, u32 flags)
  48 {
  49         u32 status;
  50         u32 counter = AES_OP_TIMEOUT;
  51
  52         iowrite32(virt_to_phys(src), _iobase + AES_SOURCEA_REG);
  53         iowrite32(virt_to_phys(dst), _iobase + AES_DSTA_REG);
  54         iowrite32(len,  _iobase + AES_LENA_REG);
  55
  56         /* Start the operation */
  57         iowrite32(AES_CTRL_START | flags, _iobase + AES_CTRLA_REG);
  58
  59         do {
  60                 status = ioread32(_iobase + AES_INTR_REG);
  61                 cpu_relax();
  62         } while(!(status & AES_INTRA_PENDING) && --counter);
  63
  64         /* Clear the event */
  65         iowrite32((status & 0xFF) | AES_INTRA_PENDING, _iobase + AES_INTR_REG);
  66         return counter ? 0 : 1;
  67 }
  68
  69 static unsigned int
  70 geode_aes_crypt(struct geode_aes_op *op)
  71 {
  72         u32 flags = 0;
  73         unsigned long iflags;
  74         int ret;
  75
  76         if (op->len == 0)
  77                 return 0;
  78
  79         /* If the source and destination is the same, then
  80          * we need to turn on the coherent flags, otherwise
  81          * we don't need to worry
  82          */
  83
  84         flags |= (AES_CTRL_DCA | AES_CTRL_SCA);
  85
  86         if (op->dir == AES_DIR_ENCRYPT)
  87                 flags |= AES_CTRL_ENCRYPT;
  88
  89         /* Start the critical section */
  90
  91         spin_lock_irqsave(&lock, iflags);
  92
  93         if (op->mode == AES_MODE_CBC) {
  94                 flags |= AES_CTRL_CBC;
  95                 _writefield(AES_WRITEIV0_REG, op->iv);
  96         }
  97
  98         if (!(op->flags & AES_FLAGS_HIDDENKEY)) {
  99                 flags |= AES_CTRL_WRKEY;
 100                 _writefield(AES_WRITEKEY0_REG, op->key);
 101         }
 102
 103         ret = do_crypt(op->src, op->dst, op->len, flags);
 104         BUG_ON(ret);
 105
 106         if (op->mode == AES_MODE_CBC)
 107                 _readfield(AES_WRITEIV0_REG, op->iv);
 108
 109         spin_unlock_irqrestore(&lock, iflags);
 110
 111         return op->len;
 112 }
 113
 114 /* CRYPTO-API Functions */
 115
 116 static int geode_setkey_cip(struct crypto_tfm *tfm, const u8 *key,
 117                 unsigned int len)
 118 {
 119         struct geode_aes_op *op = crypto_tfm_ctx(tfm);
 120         unsigned int ret;
 121
 122         op->keylen = len;
 123
 124         if (len == AES_KEYSIZE_128) {
 125                 memcpy(op->key, key, len);
 126                 return 0;
 127         }
 128
 129         if (len != AES_KEYSIZE_192 && len != AES_KEYSIZE_256) {
 130                 /* not supported at all */
 131                 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 132                 return -EINVAL;
 133         }
 134
 135         /*
 136          * The requested key size is not supported by HW, do a fallback
 137          */
 138         op->fallback.blk->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
 139         op->fallback.blk->base.crt_flags |= (tfm->crt_flags & CRYPTO_TFM_REQ_MASK);
 140
 141         ret = crypto_cipher_setkey(op->fallback.cip, key, len);
 142         if (ret) {
 143                 tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
 144                 tfm->crt_flags |= (op->fallback.blk->base.crt_flags & CRYPTO_TFM_RES_MASK);
 145         }
 146         return ret;
 147 }
 148
 149 static int geode_setkey_blk(struct crypto_tfm *tfm, const u8 *key,
 150                 unsigned int len)
 151 {
 152         struct geode_aes_op *op = crypto_tfm_ctx(tfm);
 153         unsigned int ret;
 154
 155         op->keylen = len;
 156
 157         if (len == AES_KEYSIZE_128) {
 158                 memcpy(op->key, key, len);
 159                 return 0;
 160         }
 161
 162         if (len != AES_KEYSIZE_192 && len != AES_KEYSIZE_256) {
 163                 /* not supported at all */
 164                 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 165                 return -EINVAL;
 166         }
 167
 168         /*
 169          * The requested key size is not supported by HW, do a fallback
 170          */
 171         op->fallback.blk->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
 172         op->fallback.blk->base.crt_flags |= (tfm->crt_flags & CRYPTO_TFM_REQ_MASK);
 173
 174         ret = crypto_blkcipher_setkey(op->fallback.blk, key, len);
 175         if (ret) {
 176                 tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
 177                 tfm->crt_flags |= (op->fallback.blk->base.crt_flags & CRYPTO_TFM_RES_MASK);
 178         }
 179         return ret;
 180 }
 181
 182 static int fallback_blk_dec(struct blkcipher_desc *desc,
 183                 struct scatterlist *dst, struct scatterlist *src,
 184                 unsigned int nbytes)
 185 {
 186         unsigned int ret;
 187         struct crypto_blkcipher *tfm;
 188         struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
 189
 190         tfm = desc->tfm;
 191         desc->tfm = op->fallback.blk;
 192
 193         ret = crypto_blkcipher_decrypt(desc, dst, src, nbytes);
 194
 195         desc->tfm = tfm;
 196         return ret;
 197 }
 198 static int fallback_blk_enc(struct blkcipher_desc *desc,
 199                 struct scatterlist *dst, struct scatterlist *src,
 200                 unsigned int nbytes)
 201 {
 202         unsigned int ret;
 203         struct crypto_blkcipher *tfm;
 204         struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
 205
 206         tfm = desc->tfm;
 207         desc->tfm = op->fallback.blk;
 208
 209         ret = crypto_blkcipher_encrypt(desc, dst, src, nbytes);
 210
 211         desc->tfm = tfm;
 212         return ret;
 213 }
 214
 215 static void
 216 geode_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
 217 {
 218         struct geode_aes_op *op = crypto_tfm_ctx(tfm);
 219
 220         if (unlikely(op->keylen != AES_KEYSIZE_128)) {
 221                 crypto_cipher_encrypt_one(op->fallback.cip, out, in);
 222                 return;
 223         }
 224
 225         op->src = (void *) in;
 226         op->dst = (void *) out;
 227         op->mode = AES_MODE_ECB;
 228         op->flags = 0;
 229         op->len = AES_MIN_BLOCK_SIZE;
 230         op->dir = AES_DIR_ENCRYPT;
 231
 232         geode_aes_crypt(op);
 233 }
 234
 235
 236 static void
 237 geode_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
 238 {
 239         struct geode_aes_op *op = crypto_tfm_ctx(tfm);
 240
 241         if (unlikely(op->keylen != AES_KEYSIZE_128)) {
 242                 crypto_cipher_decrypt_one(op->fallback.cip, out, in);
 243                 return;
 244         }
 245
 246         op->src = (void *) in;
 247         op->dst = (void *) out;
 248         op->mode = AES_MODE_ECB;
 249         op->flags = 0;
 250         op->len = AES_MIN_BLOCK_SIZE;
 251         op->dir = AES_DIR_DECRYPT;
 252
 253         geode_aes_crypt(op);
 254 }
 255
 256 static int fallback_init_cip(struct crypto_tfm *tfm)
 257 {
 258         const char *name = tfm->__crt_alg->cra_name;
 259         struct geode_aes_op *op = crypto_tfm_ctx(tfm);
 260
 261         op->fallback.cip = crypto_alloc_cipher(name, 0,
 262                                 CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
 263
 264         if (IS_ERR(op->fallback.cip)) {
 265                 printk(KERN_ERR "Error allocating fallback algo %s\n", name);
 266                 return PTR_ERR(op->fallback.blk);
 267         }
 268
 269         return 0;
 270 }
 271
 272 static void fallback_exit_cip(struct crypto_tfm *tfm)
 273 {
 274         struct geode_aes_op *op = crypto_tfm_ctx(tfm);
 275
 276         crypto_free_cipher(op->fallback.cip);
 277         op->fallback.cip = NULL;
 278 }
 279
 280 static struct crypto_alg geode_alg = {
 281         .cra_name                       =       "aes",
 282         .cra_driver_name        =       "geode-aes",
 283         .cra_priority           =       300,
 284         .cra_alignmask          =       15,
 285         .cra_flags                      =       CRYPTO_ALG_TYPE_CIPHER |
 286                                                         CRYPTO_ALG_NEED_FALLBACK,
 287         .cra_init                       =       fallback_init_cip,
 288         .cra_exit                       =       fallback_exit_cip,
 289         .cra_blocksize          =       AES_MIN_BLOCK_SIZE,
 290         .cra_ctxsize            =       sizeof(struct geode_aes_op),
 291         .cra_module                     =       THIS_MODULE,
 292         .cra_list                       =       LIST_HEAD_INIT(geode_alg.cra_list),
 293         .cra_u                          =       {
 294                 .cipher =       {
 295                         .cia_min_keysize        =       AES_MIN_KEY_SIZE,
 296                         .cia_max_keysize        =       AES_MAX_KEY_SIZE,
 297                         .cia_setkey                     =       geode_setkey_cip,
 298                         .cia_encrypt            =       geode_encrypt,
 299                         .cia_decrypt            =       geode_decrypt
 300                 }
 301         }
 302 };
 303
 304 static int
 305 geode_cbc_decrypt(struct blkcipher_desc *desc,
 306                   struct scatterlist *dst, struct scatterlist *src,
 307                   unsigned int nbytes)
 308 {
 309         struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
 310         struct blkcipher_walk walk;
 311         int err, ret;
 312
 313         if (unlikely(op->keylen != AES_KEYSIZE_128))
 314                 return fallback_blk_dec(desc, dst, src, nbytes);
 315
 316         blkcipher_walk_init(&walk, dst, src, nbytes);
 317         err = blkcipher_walk_virt(desc, &walk);
 318         memcpy(op->iv, walk.iv, AES_IV_LENGTH);
 319
 320         while((nbytes = walk.nbytes)) {
 321                 op->src = walk.src.virt.addr,
 322                 op->dst = walk.dst.virt.addr;
 323                 op->mode = AES_MODE_CBC;
 324                 op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
 325                 op->dir = AES_DIR_DECRYPT;
 326
 327                 ret = geode_aes_crypt(op);
 328
 329                 nbytes -= ret;
 330                 err = blkcipher_walk_done(desc, &walk, nbytes);
 331         }
 332
 333         memcpy(walk.iv, op->iv, AES_IV_LENGTH);
 334         return err;
 335 }
 336
 337 static int
 338 geode_cbc_encrypt(struct blkcipher_desc *desc,
 339                   struct scatterlist *dst, struct scatterlist *src,
 340                   unsigned int nbytes)
 341 {
 342         struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
 343         struct blkcipher_walk walk;
 344         int err, ret;
 345
 346         if (unlikely(op->keylen != AES_KEYSIZE_128))
 347                 return fallback_blk_enc(desc, dst, src, nbytes);
 348
 349         blkcipher_walk_init(&walk, dst, src, nbytes);
 350         err = blkcipher_walk_virt(desc, &walk);
 351         memcpy(op->iv, walk.iv, AES_IV_LENGTH);
 352
 353         while((nbytes = walk.nbytes)) {
 354                 op->src = walk.src.virt.addr,
 355                 op->dst = walk.dst.virt.addr;
 356                 op->mode = AES_MODE_CBC;
 357                 op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
 358                 op->dir = AES_DIR_ENCRYPT;
 359
 360                 ret = geode_aes_crypt(op);
 361                 nbytes -= ret;
 362                 err = blkcipher_walk_done(desc, &walk, nbytes);
 363         }
 364
 365         memcpy(walk.iv, op->iv, AES_IV_LENGTH);
 366         return err;
 367 }
 368
 369 static int fallback_init_blk(struct crypto_tfm *tfm)
 370 {
 371         const char *name = tfm->__crt_alg->cra_name;
 372         struct geode_aes_op *op = crypto_tfm_ctx(tfm);
 373
 374         op->fallback.blk = crypto_alloc_blkcipher(name, 0,
 375                         CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
 376
 377         if (IS_ERR(op->fallback.blk)) {
 378                 printk(KERN_ERR "Error allocating fallback algo %s\n", name);
 379                 return PTR_ERR(op->fallback.blk);
 380         }
 381
 382         return 0;
 383 }
 384
 385 static void fallback_exit_blk(struct crypto_tfm *tfm)
 386 {
 387         struct geode_aes_op *op = crypto_tfm_ctx(tfm);
 388
 389         crypto_free_blkcipher(op->fallback.blk);
 390         op->fallback.blk = NULL;
 391 }
 392
 393 static struct crypto_alg geode_cbc_alg = {
 394         .cra_name               =       "cbc(aes)",
 395         .cra_driver_name        =       "cbc-aes-geode",
 396         .cra_priority           =       400,
 397         .cra_flags                      =       CRYPTO_ALG_TYPE_BLKCIPHER |
 398                                                         CRYPTO_ALG_NEED_FALLBACK,
 399         .cra_init                       =       fallback_init_blk,
 400         .cra_exit                       =       fallback_exit_blk,
 401         .cra_blocksize          =       AES_MIN_BLOCK_SIZE,
 402         .cra_ctxsize            =       sizeof(struct geode_aes_op),
 403         .cra_alignmask          =       15,
 404         .cra_type                       =       &crypto_blkcipher_type,
 405         .cra_module                     =       THIS_MODULE,
 406         .cra_list                       =       LIST_HEAD_INIT(geode_cbc_alg.cra_list),
 407         .cra_u                          =       {
 408                 .blkcipher      =       {
 409                         .min_keysize    =       AES_MIN_KEY_SIZE,
 410                         .max_keysize    =       AES_MAX_KEY_SIZE,
 411                         .setkey                 =       geode_setkey_blk,
 412                         .encrypt                =       geode_cbc_encrypt,
 413                         .decrypt                =       geode_cbc_decrypt,
 414                         .ivsize                 =       AES_IV_LENGTH,
 415                 }
 416         }
 417 };
 418
 419 static int
 420 geode_ecb_decrypt(struct blkcipher_desc *desc,
 421                   struct scatterlist *dst, struct scatterlist *src,
 422                   unsigned int nbytes)
 423 {
 424         struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
 425         struct blkcipher_walk walk;
 426         int err, ret;
 427
 428         if (unlikely(op->keylen != AES_KEYSIZE_128))
 429                 return fallback_blk_dec(desc, dst, src, nbytes);
 430
 431         blkcipher_walk_init(&walk, dst, src, nbytes);
 432         err = blkcipher_walk_virt(desc, &walk);
 433
 434         while((nbytes = walk.nbytes)) {
 435                 op->src = walk.src.virt.addr,
 436                 op->dst = walk.dst.virt.addr;
 437                 op->mode = AES_MODE_ECB;
 438                 op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
 439                 op->dir = AES_DIR_DECRYPT;
 440
 441                 ret = geode_aes_crypt(op);
 442                 nbytes -= ret;
 443                 err = blkcipher_walk_done(desc, &walk, nbytes);
 444         }
 445
 446         return err;
 447 }
 448
 449 static int
 450 geode_ecb_encrypt(struct blkcipher_desc *desc,
 451                   struct scatterlist *dst, struct scatterlist *src,
 452                   unsigned int nbytes)
 453 {
 454         struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
 455         struct blkcipher_walk walk;
 456         int err, ret;
 457
 458         if (unlikely(op->keylen != AES_KEYSIZE_128))
 459                 return fallback_blk_enc(desc, dst, src, nbytes);
 460
 461         blkcipher_walk_init(&walk, dst, src, nbytes);
 462         err = blkcipher_walk_virt(desc, &walk);
 463
 464         while((nbytes = walk.nbytes)) {
 465                 op->src = walk.src.virt.addr,
 466                 op->dst = walk.dst.virt.addr;
 467                 op->mode = AES_MODE_ECB;
 468                 op->len = nbytes - (nbytes % AES_MIN_BLOCK_SIZE);
 469                 op->dir = AES_DIR_ENCRYPT;
 470
 471                 ret = geode_aes_crypt(op);
 472                 nbytes -= ret;
 473                 ret =  blkcipher_walk_done(desc, &walk, nbytes);
 474         }
 475
 476         return err;
 477 }
 478
 479 static struct crypto_alg geode_ecb_alg = {
 480         .cra_name                       =       "ecb(aes)",
 481         .cra_driver_name        =       "ecb-aes-geode",
 482         .cra_priority           =       400,
 483         .cra_flags                      =       CRYPTO_ALG_TYPE_BLKCIPHER |
 484                                                         CRYPTO_ALG_NEED_FALLBACK,
 485         .cra_init                       =       fallback_init_blk,
 486         .cra_exit                       =       fallback_exit_blk,
 487         .cra_blocksize          =       AES_MIN_BLOCK_SIZE,
 488         .cra_ctxsize            =       sizeof(struct geode_aes_op),
 489         .cra_alignmask          =       15,
 490         .cra_type                       =       &crypto_blkcipher_type,
 491         .cra_module                     =       THIS_MODULE,
 492         .cra_list                       =       LIST_HEAD_INIT(geode_ecb_alg.cra_list),
 493         .cra_u                          =       {
 494                 .blkcipher      =       {
 495                         .min_keysize    =       AES_MIN_KEY_SIZE,
 496                         .max_keysize    =       AES_MAX_KEY_SIZE,
 497                         .setkey                 =       geode_setkey_blk,
 498                         .encrypt                =       geode_ecb_encrypt,
 499                         .decrypt                =       geode_ecb_decrypt,
 500                 }
 501         }
 502 };
 503
 504 static void
 505 geode_aes_remove(struct pci_dev *dev)
 506 {
 507         crypto_unregister_alg(&geode_alg);
 508         crypto_unregister_alg(&geode_ecb_alg);
 509         crypto_unregister_alg(&geode_cbc_alg);
 510
 511         pci_iounmap(dev, _iobase);
 512         _iobase = NULL;
 513
 514         pci_release_regions(dev);
 515         pci_disable_device(dev);
 516 }
 517
 518
 519 static int
 520 geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id)
 521 {
 522         int ret;
 523
 524         if ((ret = pci_enable_device(dev)))
 525                 return ret;
 526
 527         if ((ret = pci_request_regions(dev, "geode-aes")))
 528                 goto eenable;
 529
 530         _iobase = pci_iomap(dev, 0, 0);
 531
 532         if (_iobase == NULL) {
 533                 ret = -ENOMEM;
 534                 goto erequest;
 535         }
 536
 537         spin_lock_init(&lock);
 538
 539         /* Clear any pending activity */
 540         iowrite32(AES_INTR_PENDING | AES_INTR_MASK, _iobase + AES_INTR_REG);
 541
 542         if ((ret = crypto_register_alg(&geode_alg)))
 543                 goto eiomap;
 544
 545         if ((ret = crypto_register_alg(&geode_ecb_alg)))
 546                 goto ealg;
 547
 548         if ((ret = crypto_register_alg(&geode_cbc_alg)))
 549                 goto eecb;
 550
 551         printk(KERN_NOTICE "geode-aes: GEODE AES engine enabled.\n");
 552         return 0;
 553
 554  eecb:
 555         crypto_unregister_alg(&geode_ecb_alg);
 556
 557  ealg:
 558         crypto_unregister_alg(&geode_alg);
 559
 560  eiomap:
 561         pci_iounmap(dev, _iobase);
 562
 563  erequest:
 564         pci_release_regions(dev);
 565
 566  eenable:
 567         pci_disable_device(dev);
 568
 569         printk(KERN_ERR "geode-aes:  GEODE AES initialization failed.\n");
 570         return ret;
 571 }
 572
 573 static struct pci_device_id geode_aes_tbl[] = {
 574         { PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_LX_AES, PCI_ANY_ID, PCI_ANY_ID} ,
 575         { 0, }
 576 };
 577
 578 MODULE_DEVICE_TABLE(pci, geode_aes_tbl);
 579
 580 static struct pci_driver geode_aes_driver = {
 581         .name = "Geode LX AES",
 582         .id_table = geode_aes_tbl,
 583         .probe = geode_aes_probe,
 584         .remove = __devexit_p(geode_aes_remove)
 585 };
 586
 587 static int __init
 588 geode_aes_init(void)
 589 {
 590         return pci_register_driver(&geode_aes_driver);
 591 }
 592
 593 static void __exit
 594 geode_aes_exit(void)
 595 {
 596         pci_unregister_driver(&geode_aes_driver);
 597 }
 598
 599 MODULE_AUTHOR("Advanced Micro Devices, Inc.");
 600 MODULE_DESCRIPTION("Geode LX Hardware AES driver");
 601 MODULE_LICENSE("GPL");
 602
 603 module_init(geode_aes_init);
 604 module_exit(geode_aes_exit);