crypto/gcm.c

   1 /*
   2  * GCM: Galois/Counter Mode.
   3  *
   4  * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 as published
   8  * by the Free Software Foundation.
   9  */
  10
  11 #include <crypto/gf128mul.h>
  12 #include <crypto/internal/skcipher.h>
  13 #include <crypto/scatterwalk.h>
  14 #include <linux/completion.h>
  15 #include <linux/err.h>
  16 #include <linux/init.h>
  17 #include <linux/kernel.h>
  18 #include <linux/module.h>
  19 #include <linux/slab.h>
  20
  21 struct gcm_instance_ctx {
  22         struct crypto_skcipher_spawn ctr;
  23 };
  24
  25 struct crypto_gcm_ctx {
  26         struct crypto_ablkcipher *ctr;
  27         struct gf128mul_4k *gf128;
  28 };
  29
  30 struct crypto_gcm_ghash_ctx {
  31         u32 bytes;
  32         u32 flags;
  33         struct gf128mul_4k *gf128;
  34         u8 buffer[16];
  35 };
  36
  37 struct crypto_gcm_req_priv_ctx {
  38         u8 auth_tag[16];
  39         u8 iauth_tag[16];
  40         struct scatterlist src[2];
  41         struct scatterlist dst[2];
  42         struct crypto_gcm_ghash_ctx ghash;
  43         struct ablkcipher_request abreq;
  44 };
  45
  46 struct crypto_gcm_setkey_result {
  47         int err;
  48         struct completion completion;
  49 };
  50
  51 static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx(
  52         struct aead_request *req)
  53 {
  54         unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));
  55
  56         return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1);
  57 }
  58
  59 static void crypto_gcm_ghash_init(struct crypto_gcm_ghash_ctx *ctx, u32 flags,
  60                                   struct gf128mul_4k *gf128)
  61 {
  62         ctx->bytes = 0;
  63         ctx->flags = flags;
  64         ctx->gf128 = gf128;
  65         memset(ctx->buffer, 0, 16);
  66 }
  67
  68 static void crypto_gcm_ghash_update(struct crypto_gcm_ghash_ctx *ctx,
  69                                     const u8 *src, unsigned int srclen)
  70 {
  71         u8 *dst = ctx->buffer;
  72
  73         if (ctx->bytes) {
  74                 int n = min(srclen, ctx->bytes);
  75                 u8 *pos = dst + (16 - ctx->bytes);
  76
  77                 ctx->bytes -= n;
  78                 srclen -= n;
  79
  80                 while (n--)
  81                         *pos++ ^= *src++;
  82
  83                 if (!ctx->bytes)
  84                         gf128mul_4k_lle((be128 *)dst, ctx->gf128);
  85         }
  86
  87         while (srclen >= 16) {
  88                 crypto_xor(dst, src, 16);
  89                 gf128mul_4k_lle((be128 *)dst, ctx->gf128);
  90                 src += 16;
  91                 srclen -= 16;
  92         }
  93
  94         if (srclen) {
  95                 ctx->bytes = 16 - srclen;
  96                 while (srclen--)
  97                         *dst++ ^= *src++;
  98         }
  99 }
 100
 101 static void crypto_gcm_ghash_update_sg(struct crypto_gcm_ghash_ctx *ctx,
 102                                        struct scatterlist *sg, int len)
 103 {
 104         struct scatter_walk walk;
 105         u8 *src;
 106         int n;
 107
 108         if (!len)
 109                 return;
 110
 111         scatterwalk_start(&walk, sg);
 112
 113         while (len) {
 114                 n = scatterwalk_clamp(&walk, len);
 115
 116                 if (!n) {
 117                         scatterwalk_start(&walk, scatterwalk_sg_next(walk.sg));
 118                         n = scatterwalk_clamp(&walk, len);
 119                 }
 120
 121                 src = scatterwalk_map(&walk, 0);
 122
 123                 crypto_gcm_ghash_update(ctx, src, n);
 124                 len -= n;
 125
 126                 scatterwalk_unmap(src, 0);
 127                 scatterwalk_advance(&walk, n);
 128                 scatterwalk_done(&walk, 0, len);
 129                 if (len)
 130                         crypto_yield(ctx->flags);
 131         }
 132 }
 133
 134 static void crypto_gcm_ghash_flush(struct crypto_gcm_ghash_ctx *ctx)
 135 {
 136         u8 *dst = ctx->buffer;
 137
 138         if (ctx->bytes) {
 139                 u8 *tmp = dst + (16 - ctx->bytes);
 140
 141                 while (ctx->bytes--)
 142                         *tmp++ ^= 0;
 143
 144                 gf128mul_4k_lle((be128 *)dst, ctx->gf128);
 145         }
 146
 147         ctx->bytes = 0;
 148 }
 149
 150 static void crypto_gcm_ghash_final_xor(struct crypto_gcm_ghash_ctx *ctx,
 151                                        unsigned int authlen,
 152                                        unsigned int cryptlen, u8 *dst)
 153 {
 154         u8 *buf = ctx->buffer;
 155         u128 lengths;
 156
 157         lengths.a = cpu_to_be64(authlen * 8);
 158         lengths.b = cpu_to_be64(cryptlen * 8);
 159
 160         crypto_gcm_ghash_flush(ctx);
 161         crypto_xor(buf, (u8 *)&lengths, 16);
 162         gf128mul_4k_lle((be128 *)buf, ctx->gf128);
 163         crypto_xor(dst, buf, 16);
 164 }
 165
 166 static void crypto_gcm_setkey_done(struct crypto_async_request *req, int err)
 167 {
 168         struct crypto_gcm_setkey_result *result = req->data;
 169
 170         if (err == -EINPROGRESS)
 171                 return;
 172
 173         result->err = err;
 174         complete(&result->completion);
 175 }
 176
 177 static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key,
 178                              unsigned int keylen)
 179 {
 180         struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
 181         struct crypto_ablkcipher *ctr = ctx->ctr;
 182         struct {
 183                 be128 hash;
 184                 u8 iv[8];
 185
 186                 struct crypto_gcm_setkey_result result;
 187
 188                 struct scatterlist sg[1];
 189                 struct ablkcipher_request req;
 190         } *data;
 191         int err;
 192
 193         crypto_ablkcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
 194         crypto_ablkcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
 195                                    CRYPTO_TFM_REQ_MASK);
 196
 197         err = crypto_ablkcipher_setkey(ctr, key, keylen);
 198         if (err)
 199                 return err;
 200
 201         crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctr) &
 202                                        CRYPTO_TFM_RES_MASK);
 203
 204         data = kzalloc(sizeof(*data) + crypto_ablkcipher_reqsize(ctr),
 205                        GFP_KERNEL);
 206         if (!data)
 207                 return -ENOMEM;
 208
 209         init_completion(&data->result.completion);
 210         sg_init_one(data->sg, &data->hash, sizeof(data->hash));
 211         ablkcipher_request_set_tfm(&data->req, ctr);
 212         ablkcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |
 213                                                     CRYPTO_TFM_REQ_MAY_BACKLOG,
 214                                         crypto_gcm_setkey_done,
 215                                         &data->result);
 216         ablkcipher_request_set_crypt(&data->req, data->sg, data->sg,
 217                                      sizeof(data->hash), data->iv);
 218
 219         err = crypto_ablkcipher_encrypt(&data->req);
 220         if (err == -EINPROGRESS || err == -EBUSY) {
 221                 err = wait_for_completion_interruptible(
 222                         &data->result.completion);
 223                 if (!err)
 224                         err = data->result.err;
 225         }
 226
 227         if (err)
 228                 goto out;
 229
 230         if (ctx->gf128 != NULL)
 231                 gf128mul_free_4k(ctx->gf128);
 232
 233         ctx->gf128 = gf128mul_init_4k_lle(&data->hash);
 234
 235         if (ctx->gf128 == NULL)
 236                 err = -ENOMEM;
 237
 238 out:
 239         kfree(data);
 240         return err;
 241 }
 242
 243 static void crypto_gcm_init_crypt(struct ablkcipher_request *ablk_req,
 244                                   struct aead_request *req,
 245                                   unsigned int cryptlen)
 246 {
 247         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 248         struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
 249         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 250         u32 flags = req->base.tfm->crt_flags;
 251         struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
 252         struct scatterlist *dst;
 253         __be32 counter = cpu_to_be32(1);
 254
 255         memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag));
 256         memcpy(req->iv + 12, &counter, 4);
 257
 258         sg_init_table(pctx->src, 2);
 259         sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag));
 260         scatterwalk_sg_chain(pctx->src, 2, req->src);
 261
 262         dst = pctx->src;
 263         if (req->src != req->dst) {
 264                 sg_init_table(pctx->dst, 2);
 265                 sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag));
 266                 scatterwalk_sg_chain(pctx->dst, 2, req->dst);
 267                 dst = pctx->dst;
 268         }
 269
 270         ablkcipher_request_set_tfm(ablk_req, ctx->ctr);
 271         ablkcipher_request_set_crypt(ablk_req, pctx->src, dst,
 272                                      cryptlen + sizeof(pctx->auth_tag),
 273                                      req->iv);
 274
 275         crypto_gcm_ghash_init(ghash, flags, ctx->gf128);
 276
 277         crypto_gcm_ghash_update_sg(ghash, req->assoc, req->assoclen);
 278         crypto_gcm_ghash_flush(ghash);
 279 }
 280
 281 static int crypto_gcm_hash(struct aead_request *req)
 282 {
 283         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 284         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 285         u8 *auth_tag = pctx->auth_tag;
 286         struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
 287
 288         crypto_gcm_ghash_update_sg(ghash, req->dst, req->cryptlen);
 289         crypto_gcm_ghash_final_xor(ghash, req->assoclen, req->cryptlen,
 290                                    auth_tag);
 291
 292         scatterwalk_map_and_copy(auth_tag, req->dst, req->cryptlen,
 293                                  crypto_aead_authsize(aead), 1);
 294         return 0;
 295 }
 296
 297 static void crypto_gcm_encrypt_done(struct crypto_async_request *areq, int err)
 298 {
 299         struct aead_request *req = areq->data;
 300
 301         if (!err)
 302                 err = crypto_gcm_hash(req);
 303
 304         aead_request_complete(req, err);
 305 }
 306
 307 static int crypto_gcm_encrypt(struct aead_request *req)
 308 {
 309         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 310         struct ablkcipher_request *abreq = &pctx->abreq;
 311         int err;
 312
 313         crypto_gcm_init_crypt(abreq, req, req->cryptlen);
 314         ablkcipher_request_set_callback(abreq, aead_request_flags(req),
 315                                         crypto_gcm_encrypt_done, req);
 316
 317         err = crypto_ablkcipher_encrypt(abreq);
 318         if (err)
 319                 return err;
 320
 321         return crypto_gcm_hash(req);
 322 }
 323
 324 static int crypto_gcm_verify(struct aead_request *req)
 325 {
 326         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 327         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 328         struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
 329         u8 *auth_tag = pctx->auth_tag;
 330         u8 *iauth_tag = pctx->iauth_tag;
 331         unsigned int authsize = crypto_aead_authsize(aead);
 332         unsigned int cryptlen = req->cryptlen - authsize;
 333
 334         crypto_gcm_ghash_final_xor(ghash, req->assoclen, cryptlen, auth_tag);
 335
 336         authsize = crypto_aead_authsize(aead);
 337         scatterwalk_map_and_copy(iauth_tag, req->src, cryptlen, authsize, 0);
 338         return memcmp(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0;
 339 }
 340
 341 static void crypto_gcm_decrypt_done(struct crypto_async_request *areq, int err)
 342 {
 343         struct aead_request *req = areq->data;
 344
 345         if (!err)
 346                 err = crypto_gcm_verify(req);
 347
 348         aead_request_complete(req, err);
 349 }
 350
 351 static int crypto_gcm_decrypt(struct aead_request *req)
 352 {
 353         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 354         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 355         struct ablkcipher_request *abreq = &pctx->abreq;
 356         struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
 357         unsigned int cryptlen = req->cryptlen;
 358         unsigned int authsize = crypto_aead_authsize(aead);
 359         int err;
 360
 361         if (cryptlen < authsize)
 362                 return -EINVAL;
 363         cryptlen -= authsize;
 364
 365         crypto_gcm_init_crypt(abreq, req, cryptlen);
 366         ablkcipher_request_set_callback(abreq, aead_request_flags(req),
 367                                         crypto_gcm_decrypt_done, req);
 368
 369         crypto_gcm_ghash_update_sg(ghash, req->src, cryptlen);
 370
 371         err = crypto_ablkcipher_decrypt(abreq);
 372         if (err)
 373                 return err;
 374
 375         return crypto_gcm_verify(req);
 376 }
 377
 378 static int crypto_gcm_init_tfm(struct crypto_tfm *tfm)
 379 {
 380         struct crypto_instance *inst = (void *)tfm->__crt_alg;
 381         struct gcm_instance_ctx *ictx = crypto_instance_ctx(inst);
 382         struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
 383         struct crypto_ablkcipher *ctr;
 384         unsigned long align;
 385         int err;
 386
 387         ctr = crypto_spawn_skcipher(&ictx->ctr);
 388         err = PTR_ERR(ctr);
 389         if (IS_ERR(ctr))
 390                 return err;
 391
 392         ctx->ctr = ctr;
 393         ctx->gf128 = NULL;
 394
 395         align = crypto_tfm_alg_alignmask(tfm);
 396         align &= ~(crypto_tfm_ctx_alignment() - 1);
 397         tfm->crt_aead.reqsize = align +
 398                                 sizeof(struct crypto_gcm_req_priv_ctx) +
 399                                 crypto_ablkcipher_reqsize(ctr);
 400
 401         return 0;
 402 }
 403
 404 static void crypto_gcm_exit_tfm(struct crypto_tfm *tfm)
 405 {
 406         struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
 407
 408         if (ctx->gf128 != NULL)
 409                 gf128mul_free_4k(ctx->gf128);
 410
 411         crypto_free_ablkcipher(ctx->ctr);
 412 }
 413
 414 static struct crypto_instance *crypto_gcm_alloc_common(struct rtattr **tb,
 415                                                        const char *full_name,
 416                                                        const char *ctr_name)
 417 {
 418         struct crypto_attr_type *algt;
 419         struct crypto_instance *inst;
 420         struct crypto_alg *ctr;
 421         struct gcm_instance_ctx *ctx;
 422         int err;
 423
 424         algt = crypto_get_attr_type(tb);
 425         err = PTR_ERR(algt);
 426         if (IS_ERR(algt))
 427                 return ERR_PTR(err);
 428
 429         if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
 430                 return ERR_PTR(-EINVAL);
 431
 432         inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
 433         if (!inst)
 434                 return ERR_PTR(-ENOMEM);
 435
 436         ctx = crypto_instance_ctx(inst);
 437         crypto_set_skcipher_spawn(&ctx->ctr, inst);
 438         err = crypto_grab_skcipher(&ctx->ctr, ctr_name, 0,
 439                                    crypto_requires_sync(algt->type,
 440                                                         algt->mask));
 441         if (err)
 442                 goto err_free_inst;
 443
 444         ctr = crypto_skcipher_spawn_alg(&ctx->ctr);
 445
 446         /* We only support 16-byte blocks. */
 447         if (ctr->cra_ablkcipher.ivsize != 16)
 448                 goto out_put_ctr;
 449
 450         /* Not a stream cipher? */
 451         err = -EINVAL;
 452         if (ctr->cra_blocksize != 1)
 453                 goto out_put_ctr;
 454
 455         err = -ENAMETOOLONG;
 456         if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
 457                      "gcm_base(%s)", ctr->cra_driver_name) >=
 458             CRYPTO_MAX_ALG_NAME)
 459                 goto out_put_ctr;
 460
 461         memcpy(inst->alg.cra_name, full_name, CRYPTO_MAX_ALG_NAME);
 462
 463         inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD;
 464         inst->alg.cra_flags |= ctr->cra_flags & CRYPTO_ALG_ASYNC;
 465         inst->alg.cra_priority = ctr->cra_priority;
 466         inst->alg.cra_blocksize = 1;
 467         inst->alg.cra_alignmask = ctr->cra_alignmask | (__alignof__(u64) - 1);
 468         inst->alg.cra_type = &crypto_aead_type;
 469         inst->alg.cra_aead.ivsize = 16;
 470         inst->alg.cra_aead.maxauthsize = 16;
 471         inst->alg.cra_ctxsize = sizeof(struct crypto_gcm_ctx);
 472         inst->alg.cra_init = crypto_gcm_init_tfm;
 473         inst->alg.cra_exit = crypto_gcm_exit_tfm;
 474         inst->alg.cra_aead.setkey = crypto_gcm_setkey;
 475         inst->alg.cra_aead.encrypt = crypto_gcm_encrypt;
 476         inst->alg.cra_aead.decrypt = crypto_gcm_decrypt;
 477
 478 out:
 479         return inst;
 480
 481 out_put_ctr:
 482         crypto_drop_skcipher(&ctx->ctr);
 483 err_free_inst:
 484         kfree(inst);
 485         inst = ERR_PTR(err);
 486         goto out;
 487 }
 488
 489 static struct crypto_instance *crypto_gcm_alloc(struct rtattr **tb)
 490 {
 491         int err;
 492         const char *cipher_name;
 493         char ctr_name[CRYPTO_MAX_ALG_NAME];
 494         char full_name[CRYPTO_MAX_ALG_NAME];
 495
 496         cipher_name = crypto_attr_alg_name(tb[1]);
 497         err = PTR_ERR(cipher_name);
 498         if (IS_ERR(cipher_name))
 499                 return ERR_PTR(err);
 500
 501         if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >=
 502             CRYPTO_MAX_ALG_NAME)
 503                 return ERR_PTR(-ENAMETOOLONG);
 504
 505         if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm(%s)", cipher_name) >=
 506             CRYPTO_MAX_ALG_NAME)
 507                 return ERR_PTR(-ENAMETOOLONG);
 508
 509         return crypto_gcm_alloc_common(tb, full_name, ctr_name);
 510 }
 511
 512 static void crypto_gcm_free(struct crypto_instance *inst)
 513 {
 514         struct gcm_instance_ctx *ctx = crypto_instance_ctx(inst);
 515
 516         crypto_drop_skcipher(&ctx->ctr);
 517         kfree(inst);
 518 }
 519
 520 static struct crypto_template crypto_gcm_tmpl = {
 521         .name = "gcm",
 522         .alloc = crypto_gcm_alloc,
 523         .free = crypto_gcm_free,
 524         .module = THIS_MODULE,
 525 };
 526
 527 static struct crypto_instance *crypto_gcm_base_alloc(struct rtattr **tb)
 528 {
 529         int err;
 530         const char *ctr_name;
 531         char full_name[CRYPTO_MAX_ALG_NAME];
 532
 533         ctr_name = crypto_attr_alg_name(tb[1]);
 534         err = PTR_ERR(ctr_name);
 535         if (IS_ERR(ctr_name))
 536                 return ERR_PTR(err);
 537
 538         if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s)",
 539                      ctr_name) >= CRYPTO_MAX_ALG_NAME)
 540                 return ERR_PTR(-ENAMETOOLONG);
 541
 542         return crypto_gcm_alloc_common(tb, full_name, ctr_name);
 543 }
 544
 545 static struct crypto_template crypto_gcm_base_tmpl = {
 546         .name = "gcm_base",
 547         .alloc = crypto_gcm_base_alloc,
 548         .free = crypto_gcm_free,
 549         .module = THIS_MODULE,
 550 };
 551
 552 static int __init crypto_gcm_module_init(void)
 553 {
 554         int err;
 555
 556         err = crypto_register_template(&crypto_gcm_base_tmpl);
 557         if (err)
 558                 goto out;
 559
 560         err = crypto_register_template(&crypto_gcm_tmpl);
 561         if (err)
 562                 goto out_undo_base;
 563
 564 out:
 565         return err;
 566
 567 out_undo_base:
 568         crypto_unregister_template(&crypto_gcm_base_tmpl);
 569         goto out;
 570 }
 571
 572 static void __exit crypto_gcm_module_exit(void)
 573 {
 574         crypto_unregister_template(&crypto_gcm_tmpl);
 575         crypto_unregister_template(&crypto_gcm_base_tmpl);
 576 }
 577
 578 module_init(crypto_gcm_module_init);
 579 module_exit(crypto_gcm_module_exit);
 580
 581 MODULE_LICENSE("GPL");
 582 MODULE_DESCRIPTION("Galois/Counter Mode");
 583 MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>");
 584 MODULE_ALIAS("gcm_base");