drivers/staging/lustre/lustre/lov/lov_dev.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19  *
  20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21  * CA 95054 USA or visit www.sun.com if you need additional information or
  22  * have any questions.
  23  *
  24  * GPL HEADER END
  25  */
  26 /*
  27  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Use is subject to license terms.
  29  *
  30  * Copyright (c) 2012, Intel Corporation.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * Implementation of cl_device and cl_device_type for LOV layer.
  37  *
  38  *   Author: Nikita Danilov <nikita.danilov@sun.com>
  39  */
  40
  41 #define DEBUG_SUBSYSTEM S_LOV
  42
  43 /* class_name2obd() */
  44 #include "../include/obd_class.h"
  45
  46 #include "lov_cl_internal.h"
  47 #include "lov_internal.h"
  48
  49
  50 struct kmem_cache *lov_lock_kmem;
  51 struct kmem_cache *lov_object_kmem;
  52 struct kmem_cache *lov_thread_kmem;
  53 struct kmem_cache *lov_session_kmem;
  54 struct kmem_cache *lov_req_kmem;
  55
  56 struct kmem_cache *lovsub_lock_kmem;
  57 struct kmem_cache *lovsub_object_kmem;
  58 struct kmem_cache *lovsub_req_kmem;
  59
  60 struct kmem_cache *lov_lock_link_kmem;
  61
  62 /** Lock class of lov_device::ld_mutex. */
  63 static struct lock_class_key cl_lov_device_mutex_class;
  64
  65 struct lu_kmem_descr lov_caches[] = {
  66         {
  67                 .ckd_cache = &lov_lock_kmem,
  68                 .ckd_name  = "lov_lock_kmem",
  69                 .ckd_size  = sizeof(struct lov_lock)
  70         },
  71         {
  72                 .ckd_cache = &lov_object_kmem,
  73                 .ckd_name  = "lov_object_kmem",
  74                 .ckd_size  = sizeof(struct lov_object)
  75         },
  76         {
  77                 .ckd_cache = &lov_thread_kmem,
  78                 .ckd_name  = "lov_thread_kmem",
  79                 .ckd_size  = sizeof(struct lov_thread_info)
  80         },
  81         {
  82                 .ckd_cache = &lov_session_kmem,
  83                 .ckd_name  = "lov_session_kmem",
  84                 .ckd_size  = sizeof(struct lov_session)
  85         },
  86         {
  87                 .ckd_cache = &lov_req_kmem,
  88                 .ckd_name  = "lov_req_kmem",
  89                 .ckd_size  = sizeof(struct lov_req)
  90         },
  91         {
  92                 .ckd_cache = &lovsub_lock_kmem,
  93                 .ckd_name  = "lovsub_lock_kmem",
  94                 .ckd_size  = sizeof(struct lovsub_lock)
  95         },
  96         {
  97                 .ckd_cache = &lovsub_object_kmem,
  98                 .ckd_name  = "lovsub_object_kmem",
  99                 .ckd_size  = sizeof(struct lovsub_object)
 100         },
 101         {
 102                 .ckd_cache = &lovsub_req_kmem,
 103                 .ckd_name  = "lovsub_req_kmem",
 104                 .ckd_size  = sizeof(struct lovsub_req)
 105         },
 106         {
 107                 .ckd_cache = &lov_lock_link_kmem,
 108                 .ckd_name  = "lov_lock_link_kmem",
 109                 .ckd_size  = sizeof(struct lov_lock_link)
 110         },
 111         {
 112                 .ckd_cache = NULL
 113         }
 114 };
 115
 116 /*****************************************************************************
 117  *
 118  * Lov transfer operations.
 119  *
 120  */
 121
 122 static void lov_req_completion(const struct lu_env *env,
 123                                const struct cl_req_slice *slice, int ioret)
 124 {
 125         struct lov_req *lr;
 126
 127         lr = cl2lov_req(slice);
 128         OBD_SLAB_FREE_PTR(lr, lov_req_kmem);
 129 }
 130
 131 static const struct cl_req_operations lov_req_ops = {
 132         .cro_completion = lov_req_completion
 133 };
 134
 135 /*****************************************************************************
 136  *
 137  * Lov device and device type functions.
 138  *
 139  */
 140
 141 static void *lov_key_init(const struct lu_context *ctx,
 142                           struct lu_context_key *key)
 143 {
 144         struct lov_thread_info *info;
 145
 146         OBD_SLAB_ALLOC_PTR_GFP(info, lov_thread_kmem, GFP_NOFS);
 147         if (info != NULL)
 148                 INIT_LIST_HEAD(&info->lti_closure.clc_list);
 149         else
 150                 info = ERR_PTR(-ENOMEM);
 151         return info;
 152 }
 153
 154 static void lov_key_fini(const struct lu_context *ctx,
 155                          struct lu_context_key *key, void *data)
 156 {
 157         struct lov_thread_info *info = data;
 158
 159         LINVRNT(list_empty(&info->lti_closure.clc_list));
 160         OBD_SLAB_FREE_PTR(info, lov_thread_kmem);
 161 }
 162
 163 struct lu_context_key lov_key = {
 164         .lct_tags = LCT_CL_THREAD,
 165         .lct_init = lov_key_init,
 166         .lct_fini = lov_key_fini
 167 };
 168
 169 static void *lov_session_key_init(const struct lu_context *ctx,
 170                                   struct lu_context_key *key)
 171 {
 172         struct lov_session *info;
 173
 174         OBD_SLAB_ALLOC_PTR_GFP(info, lov_session_kmem, GFP_NOFS);
 175         if (info == NULL)
 176                 info = ERR_PTR(-ENOMEM);
 177         return info;
 178 }
 179
 180 static void lov_session_key_fini(const struct lu_context *ctx,
 181                                  struct lu_context_key *key, void *data)
 182 {
 183         struct lov_session *info = data;
 184
 185         OBD_SLAB_FREE_PTR(info, lov_session_kmem);
 186 }
 187
 188 struct lu_context_key lov_session_key = {
 189         .lct_tags = LCT_SESSION,
 190         .lct_init = lov_session_key_init,
 191         .lct_fini = lov_session_key_fini
 192 };
 193
 194 /* type constructor/destructor: lov_type_{init,fini,start,stop}() */
 195 LU_TYPE_INIT_FINI(lov, &lov_key, &lov_session_key);
 196
 197 static struct lu_device *lov_device_fini(const struct lu_env *env,
 198                                          struct lu_device *d)
 199 {
 200         int i;
 201         struct lov_device *ld = lu2lov_dev(d);
 202
 203         LASSERT(ld->ld_lov != NULL);
 204         if (ld->ld_target == NULL)
 205                 return NULL;
 206
 207         lov_foreach_target(ld, i) {
 208                 struct lovsub_device *lsd;
 209
 210                 lsd = ld->ld_target[i];
 211                 if (lsd != NULL) {
 212                         cl_stack_fini(env, lovsub2cl_dev(lsd));
 213                         ld->ld_target[i] = NULL;
 214                 }
 215         }
 216         return NULL;
 217 }
 218
 219 static int lov_device_init(const struct lu_env *env, struct lu_device *d,
 220                            const char *name, struct lu_device *next)
 221 {
 222         struct lov_device *ld = lu2lov_dev(d);
 223         int i;
 224         int rc = 0;
 225
 226         LASSERT(d->ld_site != NULL);
 227         if (ld->ld_target == NULL)
 228                 return rc;
 229
 230         lov_foreach_target(ld, i) {
 231                 struct lovsub_device *lsd;
 232                 struct cl_device     *cl;
 233                 struct lov_tgt_desc  *desc;
 234
 235                 desc = ld->ld_lov->lov_tgts[i];
 236                 if (desc == NULL)
 237                         continue;
 238
 239                 cl = cl_type_setup(env, d->ld_site, &lovsub_device_type,
 240                                    desc->ltd_obd->obd_lu_dev);
 241                 if (IS_ERR(cl)) {
 242                         rc = PTR_ERR(cl);
 243                         break;
 244                 }
 245                 lsd = cl2lovsub_dev(cl);
 246                 lsd->acid_idx = i;
 247                 lsd->acid_super = ld;
 248                 ld->ld_target[i] = lsd;
 249         }
 250
 251         if (rc)
 252                 lov_device_fini(env, d);
 253         else
 254                 ld->ld_flags |= LOV_DEV_INITIALIZED;
 255
 256         return rc;
 257 }
 258
 259 static int lov_req_init(const struct lu_env *env, struct cl_device *dev,
 260                         struct cl_req *req)
 261 {
 262         struct lov_req *lr;
 263         int result;
 264
 265         OBD_SLAB_ALLOC_PTR_GFP(lr, lov_req_kmem, GFP_NOFS);
 266         if (lr != NULL) {
 267                 cl_req_slice_add(req, &lr->lr_cl, dev, &lov_req_ops);
 268                 result = 0;
 269         } else
 270                 result = -ENOMEM;
 271         return result;
 272 }
 273
 274 static const struct cl_device_operations lov_cl_ops = {
 275         .cdo_req_init = lov_req_init
 276 };
 277
 278 static void lov_emerg_free(struct lov_device_emerg **emrg, int nr)
 279 {
 280         int i;
 281
 282         for (i = 0; i < nr; ++i) {
 283                 struct lov_device_emerg *em;
 284
 285                 em = emrg[i];
 286                 if (em != NULL) {
 287                         LASSERT(em->emrg_page_list.pl_nr == 0);
 288                         if (em->emrg_env != NULL)
 289                                 cl_env_put(em->emrg_env, &em->emrg_refcheck);
 290                         kfree(em);
 291                 }
 292         }
 293         kfree(emrg);
 294 }
 295
 296 static struct lu_device *lov_device_free(const struct lu_env *env,
 297                                          struct lu_device *d)
 298 {
 299         struct lov_device *ld = lu2lov_dev(d);
 300         const int         nr = ld->ld_target_nr;
 301
 302         cl_device_fini(lu2cl_dev(d));
 303         kfree(ld->ld_target);
 304         if (ld->ld_emrg != NULL)
 305                 lov_emerg_free(ld->ld_emrg, nr);
 306         kfree(ld);
 307         return NULL;
 308 }
 309
 310 static void lov_cl_del_target(const struct lu_env *env, struct lu_device *dev,
 311                               __u32 index)
 312 {
 313         struct lov_device *ld = lu2lov_dev(dev);
 314
 315         if (ld->ld_target[index] != NULL) {
 316                 cl_stack_fini(env, lovsub2cl_dev(ld->ld_target[index]));
 317                 ld->ld_target[index] = NULL;
 318         }
 319 }
 320
 321 static struct lov_device_emerg **lov_emerg_alloc(int nr)
 322 {
 323         struct lov_device_emerg **emerg;
 324         int i;
 325         int result;
 326
 327         emerg = kcalloc(nr, sizeof(emerg[0]), GFP_NOFS);
 328         if (emerg == NULL)
 329                 return ERR_PTR(-ENOMEM);
 330         for (result = i = 0; i < nr && result == 0; i++) {
 331                 struct lov_device_emerg *em;
 332
 333                 em = kzalloc(sizeof(*em), GFP_NOFS);
 334                 if (em != NULL) {
 335                         emerg[i] = em;
 336                         cl_page_list_init(&em->emrg_page_list);
 337                         em->emrg_env = cl_env_alloc(&em->emrg_refcheck,
 338                                                     LCT_REMEMBER|LCT_NOREF);
 339                         if (!IS_ERR(em->emrg_env))
 340                                 em->emrg_env->le_ctx.lc_cookie = 0x2;
 341                         else {
 342                                 result = PTR_ERR(em->emrg_env);
 343                                 em->emrg_env = NULL;
 344                         }
 345                 } else
 346                         result = -ENOMEM;
 347         }
 348         if (result != 0) {
 349                 lov_emerg_free(emerg, nr);
 350                 emerg = ERR_PTR(result);
 351         }
 352         return emerg;
 353 }
 354
 355 static int lov_expand_targets(const struct lu_env *env, struct lov_device *dev)
 356 {
 357         int   result;
 358         __u32 tgt_size;
 359         __u32 sub_size;
 360
 361         result = 0;
 362         tgt_size = dev->ld_lov->lov_tgt_size;
 363         sub_size = dev->ld_target_nr;
 364         if (sub_size < tgt_size) {
 365                 struct lovsub_device    **newd;
 366                 struct lov_device_emerg **emerg;
 367                 const size_t          sz   = sizeof(newd[0]);
 368
 369                 emerg = lov_emerg_alloc(tgt_size);
 370                 if (IS_ERR(emerg))
 371                         return PTR_ERR(emerg);
 372
 373                 newd = kcalloc(tgt_size, sz, GFP_NOFS);
 374                 if (newd != NULL) {
 375                         mutex_lock(&dev->ld_mutex);
 376                         if (sub_size > 0) {
 377                                 memcpy(newd, dev->ld_target, sub_size * sz);
 378                                 kfree(dev->ld_target);
 379                         }
 380                         dev->ld_target    = newd;
 381                         dev->ld_target_nr = tgt_size;
 382
 383                         if (dev->ld_emrg != NULL)
 384                                 lov_emerg_free(dev->ld_emrg, sub_size);
 385                         dev->ld_emrg = emerg;
 386                         mutex_unlock(&dev->ld_mutex);
 387                 } else {
 388                         lov_emerg_free(emerg, tgt_size);
 389                         result = -ENOMEM;
 390                 }
 391         }
 392         return result;
 393 }
 394
 395 static int lov_cl_add_target(const struct lu_env *env, struct lu_device *dev,
 396                              __u32 index)
 397 {
 398         struct obd_device    *obd = dev->ld_obd;
 399         struct lov_device    *ld  = lu2lov_dev(dev);
 400         struct lov_tgt_desc  *tgt;
 401         struct lovsub_device *lsd;
 402         struct cl_device     *cl;
 403         int rc;
 404
 405         obd_getref(obd);
 406
 407         tgt = obd->u.lov.lov_tgts[index];
 408         LASSERT(tgt != NULL);
 409         LASSERT(tgt->ltd_obd != NULL);
 410
 411         if (!tgt->ltd_obd->obd_set_up) {
 412                 CERROR("Target %s not set up\n", obd_uuid2str(&tgt->ltd_uuid));
 413                 return -EINVAL;
 414         }
 415
 416         rc = lov_expand_targets(env, ld);
 417         if (rc == 0 && ld->ld_flags & LOV_DEV_INITIALIZED) {
 418                 LASSERT(dev->ld_site != NULL);
 419
 420                 cl = cl_type_setup(env, dev->ld_site, &lovsub_device_type,
 421                                    tgt->ltd_obd->obd_lu_dev);
 422                 if (!IS_ERR(cl)) {
 423                         lsd = cl2lovsub_dev(cl);
 424                         lsd->acid_idx = index;
 425                         lsd->acid_super = ld;
 426                         ld->ld_target[index] = lsd;
 427                 } else {
 428                         CERROR("add failed (%d), deleting %s\n", rc,
 429                                obd_uuid2str(&tgt->ltd_uuid));
 430                         lov_cl_del_target(env, dev, index);
 431                         rc = PTR_ERR(cl);
 432                 }
 433         }
 434         obd_putref(obd);
 435         return rc;
 436 }
 437
 438 static int lov_process_config(const struct lu_env *env,
 439                               struct lu_device *d, struct lustre_cfg *cfg)
 440 {
 441         struct obd_device *obd = d->ld_obd;
 442         int cmd;
 443         int rc;
 444         int gen;
 445         __u32 index;
 446
 447         obd_getref(obd);
 448
 449         cmd = cfg->lcfg_command;
 450         rc = lov_process_config_base(d->ld_obd, cfg, &index, &gen);
 451         if (rc == 0) {
 452                 switch (cmd) {
 453                 case LCFG_LOV_ADD_OBD:
 454                 case LCFG_LOV_ADD_INA:
 455                         rc = lov_cl_add_target(env, d, index);
 456                         if (rc != 0)
 457                                 lov_del_target(d->ld_obd, index, NULL, 0);
 458                         break;
 459                 case LCFG_LOV_DEL_OBD:
 460                         lov_cl_del_target(env, d, index);
 461                         break;
 462                 }
 463         }
 464         obd_putref(obd);
 465         return rc;
 466 }
 467
 468 static const struct lu_device_operations lov_lu_ops = {
 469         .ldo_object_alloc      = lov_object_alloc,
 470         .ldo_process_config    = lov_process_config,
 471 };
 472
 473 static struct lu_device *lov_device_alloc(const struct lu_env *env,
 474                                           struct lu_device_type *t,
 475                                           struct lustre_cfg *cfg)
 476 {
 477         struct lu_device *d;
 478         struct lov_device *ld;
 479         struct obd_device *obd;
 480         int rc;
 481
 482         ld = kzalloc(sizeof(*ld), GFP_NOFS);
 483         if (!ld)
 484                 return ERR_PTR(-ENOMEM);
 485
 486         cl_device_init(&ld->ld_cl, t);
 487         d = lov2lu_dev(ld);
 488         d->ld_ops       = &lov_lu_ops;
 489         ld->ld_cl.cd_ops = &lov_cl_ops;
 490
 491         mutex_init(&ld->ld_mutex);
 492         lockdep_set_class(&ld->ld_mutex, &cl_lov_device_mutex_class);
 493
 494         /* setup the LOV OBD */
 495         obd = class_name2obd(lustre_cfg_string(cfg, 0));
 496         LASSERT(obd != NULL);
 497         rc = lov_setup(obd, cfg);
 498         if (rc) {
 499                 lov_device_free(env, d);
 500                 return ERR_PTR(rc);
 501         }
 502
 503         ld->ld_lov = &obd->u.lov;
 504         return d;
 505 }
 506
 507 static const struct lu_device_type_operations lov_device_type_ops = {
 508         .ldto_init = lov_type_init,
 509         .ldto_fini = lov_type_fini,
 510
 511         .ldto_start = lov_type_start,
 512         .ldto_stop  = lov_type_stop,
 513
 514         .ldto_device_alloc = lov_device_alloc,
 515         .ldto_device_free  = lov_device_free,
 516
 517         .ldto_device_init    = lov_device_init,
 518         .ldto_device_fini    = lov_device_fini
 519 };
 520
 521 struct lu_device_type lov_device_type = {
 522         .ldt_tags     = LU_DEVICE_CL,
 523         .ldt_name     = LUSTRE_LOV_NAME,
 524         .ldt_ops      = &lov_device_type_ops,
 525         .ldt_ctx_tags = LCT_CL_THREAD
 526 };
 527 EXPORT_SYMBOL(lov_device_type);
 528
 529 /** @} lov */