drivers/nvdimm/region_devs.c

   1 /*
   2  * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of version 2 of the GNU General Public License as
   6  * published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful, but
   9  * WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  */
  13 #include <linux/scatterlist.h>
  14 #include <linux/highmem.h>
  15 #include <linux/sched.h>
  16 #include <linux/slab.h>
  17 #include <linux/pmem.h>
  18 #include <linux/sort.h>
  19 #include <linux/io.h>
  20 #include <linux/nd.h>
  21 #include "nd-core.h"
  22 #include "nd.h"
  23
  24 /*
  25  * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is
  26  * irrelevant.
  27  */
  28 #include <linux/io-64-nonatomic-hi-lo.h>
  29
  30 static DEFINE_IDA(region_ida);
  31
  32 static int nvdimm_map_flush(struct device *dev, struct nvdimm *nvdimm, int dimm,
  33                 struct nd_region_data *ndrd)
  34 {
  35         int i, j;
  36
  37         dev_dbg(dev, "%s: map %d flush address%s\n", nvdimm_name(nvdimm),
  38                         nvdimm->num_flush, nvdimm->num_flush == 1 ? "" : "es");
  39         for (i = 0; i < nvdimm->num_flush; i++) {
  40                 struct resource *res = &nvdimm->flush_wpq[i];
  41                 unsigned long pfn = PHYS_PFN(res->start);
  42                 void __iomem *flush_page;
  43
  44                 /* check if flush hints share a page */
  45                 for (j = 0; j < i; j++) {
  46                         struct resource *res_j = &nvdimm->flush_wpq[j];
  47                         unsigned long pfn_j = PHYS_PFN(res_j->start);
  48
  49                         if (pfn == pfn_j)
  50                                 break;
  51                 }
  52
  53                 if (j < i)
  54                         flush_page = (void __iomem *) ((unsigned long)
  55                                         ndrd->flush_wpq[dimm][j] & PAGE_MASK);
  56                 else
  57                         flush_page = devm_nvdimm_ioremap(dev,
  58                                         PHYS_PFN(pfn), PAGE_SIZE);
  59                 if (!flush_page)
  60                         return -ENXIO;
  61                 ndrd->flush_wpq[dimm][i] = flush_page
  62                         + (res->start & ~PAGE_MASK);
  63         }
  64
  65         return 0;
  66 }
  67
  68 int nd_region_activate(struct nd_region *nd_region)
  69 {
  70         int i;
  71         struct nd_region_data *ndrd;
  72         struct device *dev = &nd_region->dev;
  73         size_t flush_data_size = sizeof(void *);
  74
  75         nvdimm_bus_lock(&nd_region->dev);
  76         for (i = 0; i < nd_region->ndr_mappings; i++) {
  77                 struct nd_mapping *nd_mapping = &nd_region->mapping[i];
  78                 struct nvdimm *nvdimm = nd_mapping->nvdimm;
  79
  80                 /* at least one null hint slot per-dimm for the "no-hint" case */
  81                 flush_data_size += sizeof(void *);
  82                 if (!nvdimm->num_flush)
  83                         continue;
  84                 flush_data_size += nvdimm->num_flush * sizeof(void *);
  85         }
  86         nvdimm_bus_unlock(&nd_region->dev);
  87
  88         ndrd = devm_kzalloc(dev, sizeof(*ndrd) + flush_data_size, GFP_KERNEL);
  89         if (!ndrd)
  90                 return -ENOMEM;
  91         dev_set_drvdata(dev, ndrd);
  92
  93         for (i = 0; i < nd_region->ndr_mappings; i++) {
  94                 struct nd_mapping *nd_mapping = &nd_region->mapping[i];
  95                 struct nvdimm *nvdimm = nd_mapping->nvdimm;
  96                 int rc = nvdimm_map_flush(&nd_region->dev, nvdimm, i, ndrd);
  97
  98                 if (rc)
  99                         return rc;
 100         }
 101
 102         return 0;
 103 }
 104
 105 static void nd_region_release(struct device *dev)
 106 {
 107         struct nd_region *nd_region = to_nd_region(dev);
 108         u16 i;
 109
 110         for (i = 0; i < nd_region->ndr_mappings; i++) {
 111                 struct nd_mapping *nd_mapping = &nd_region->mapping[i];
 112                 struct nvdimm *nvdimm = nd_mapping->nvdimm;
 113
 114                 put_device(&nvdimm->dev);
 115         }
 116         free_percpu(nd_region->lane);
 117         ida_simple_remove(&region_ida, nd_region->id);
 118         if (is_nd_blk(dev))
 119                 kfree(to_nd_blk_region(dev));
 120         else
 121                 kfree(nd_region);
 122 }
 123
 124 static struct device_type nd_blk_device_type = {
 125         .name = "nd_blk",
 126         .release = nd_region_release,
 127 };
 128
 129 static struct device_type nd_pmem_device_type = {
 130         .name = "nd_pmem",
 131         .release = nd_region_release,
 132 };
 133
 134 static struct device_type nd_volatile_device_type = {
 135         .name = "nd_volatile",
 136         .release = nd_region_release,
 137 };
 138
 139 bool is_nd_pmem(struct device *dev)
 140 {
 141         return dev ? dev->type == &nd_pmem_device_type : false;
 142 }
 143
 144 bool is_nd_blk(struct device *dev)
 145 {
 146         return dev ? dev->type == &nd_blk_device_type : false;
 147 }
 148
 149 struct nd_region *to_nd_region(struct device *dev)
 150 {
 151         struct nd_region *nd_region = container_of(dev, struct nd_region, dev);
 152
 153         WARN_ON(dev->type->release != nd_region_release);
 154         return nd_region;
 155 }
 156 EXPORT_SYMBOL_GPL(to_nd_region);
 157
 158 struct nd_blk_region *to_nd_blk_region(struct device *dev)
 159 {
 160         struct nd_region *nd_region = to_nd_region(dev);
 161
 162         WARN_ON(!is_nd_blk(dev));
 163         return container_of(nd_region, struct nd_blk_region, nd_region);
 164 }
 165 EXPORT_SYMBOL_GPL(to_nd_blk_region);
 166
 167 void *nd_region_provider_data(struct nd_region *nd_region)
 168 {
 169         return nd_region->provider_data;
 170 }
 171 EXPORT_SYMBOL_GPL(nd_region_provider_data);
 172
 173 void *nd_blk_region_provider_data(struct nd_blk_region *ndbr)
 174 {
 175         return ndbr->blk_provider_data;
 176 }
 177 EXPORT_SYMBOL_GPL(nd_blk_region_provider_data);
 178
 179 void nd_blk_region_set_provider_data(struct nd_blk_region *ndbr, void *data)
 180 {
 181         ndbr->blk_provider_data = data;
 182 }
 183 EXPORT_SYMBOL_GPL(nd_blk_region_set_provider_data);
 184
 185 /**
 186  * nd_region_to_nstype() - region to an integer namespace type
 187  * @nd_region: region-device to interrogate
 188  *
 189  * This is the 'nstype' attribute of a region as well, an input to the
 190  * MODALIAS for namespace devices, and bit number for a nvdimm_bus to match
 191  * namespace devices with namespace drivers.
 192  */
 193 int nd_region_to_nstype(struct nd_region *nd_region)
 194 {
 195         if (is_nd_pmem(&nd_region->dev)) {
 196                 u16 i, alias;
 197
 198                 for (i = 0, alias = 0; i < nd_region->ndr_mappings; i++) {
 199                         struct nd_mapping *nd_mapping = &nd_region->mapping[i];
 200                         struct nvdimm *nvdimm = nd_mapping->nvdimm;
 201
 202                         if (nvdimm->flags & NDD_ALIASING)
 203                                 alias++;
 204                 }
 205                 if (alias)
 206                         return ND_DEVICE_NAMESPACE_PMEM;
 207                 else
 208                         return ND_DEVICE_NAMESPACE_IO;
 209         } else if (is_nd_blk(&nd_region->dev)) {
 210                 return ND_DEVICE_NAMESPACE_BLK;
 211         }
 212
 213         return 0;
 214 }
 215 EXPORT_SYMBOL(nd_region_to_nstype);
 216
 217 static ssize_t size_show(struct device *dev,
 218                 struct device_attribute *attr, char *buf)
 219 {
 220         struct nd_region *nd_region = to_nd_region(dev);
 221         unsigned long long size = 0;
 222
 223         if (is_nd_pmem(dev)) {
 224                 size = nd_region->ndr_size;
 225         } else if (nd_region->ndr_mappings == 1) {
 226                 struct nd_mapping *nd_mapping = &nd_region->mapping[0];
 227
 228                 size = nd_mapping->size;
 229         }
 230
 231         return sprintf(buf, "%llu\n", size);
 232 }
 233 static DEVICE_ATTR_RO(size);
 234
 235 static ssize_t mappings_show(struct device *dev,
 236                 struct device_attribute *attr, char *buf)
 237 {
 238         struct nd_region *nd_region = to_nd_region(dev);
 239
 240         return sprintf(buf, "%d\n", nd_region->ndr_mappings);
 241 }
 242 static DEVICE_ATTR_RO(mappings);
 243
 244 static ssize_t nstype_show(struct device *dev,
 245                 struct device_attribute *attr, char *buf)
 246 {
 247         struct nd_region *nd_region = to_nd_region(dev);
 248
 249         return sprintf(buf, "%d\n", nd_region_to_nstype(nd_region));
 250 }
 251 static DEVICE_ATTR_RO(nstype);
 252
 253 static ssize_t set_cookie_show(struct device *dev,
 254                 struct device_attribute *attr, char *buf)
 255 {
 256         struct nd_region *nd_region = to_nd_region(dev);
 257         struct nd_interleave_set *nd_set = nd_region->nd_set;
 258
 259         if (is_nd_pmem(dev) && nd_set)
 260                 /* pass, should be precluded by region_visible */;
 261         else
 262                 return -ENXIO;
 263
 264         return sprintf(buf, "%#llx\n", nd_set->cookie);
 265 }
 266 static DEVICE_ATTR_RO(set_cookie);
 267
 268 resource_size_t nd_region_available_dpa(struct nd_region *nd_region)
 269 {
 270         resource_size_t blk_max_overlap = 0, available, overlap;
 271         int i;
 272
 273         WARN_ON(!is_nvdimm_bus_locked(&nd_region->dev));
 274
 275  retry:
 276         available = 0;
 277         overlap = blk_max_overlap;
 278         for (i = 0; i < nd_region->ndr_mappings; i++) {
 279                 struct nd_mapping *nd_mapping = &nd_region->mapping[i];
 280                 struct nvdimm_drvdata *ndd = to_ndd(nd_mapping);
 281
 282                 /* if a dimm is disabled the available capacity is zero */
 283                 if (!ndd)
 284                         return 0;
 285
 286                 if (is_nd_pmem(&nd_region->dev)) {
 287                         available += nd_pmem_available_dpa(nd_region,
 288                                         nd_mapping, &overlap);
 289                         if (overlap > blk_max_overlap) {
 290                                 blk_max_overlap = overlap;
 291                                 goto retry;
 292                         }
 293                 } else if (is_nd_blk(&nd_region->dev)) {
 294                         available += nd_blk_available_dpa(nd_mapping);
 295                 }
 296         }
 297
 298         return available;
 299 }
 300
 301 static ssize_t available_size_show(struct device *dev,
 302                 struct device_attribute *attr, char *buf)
 303 {
 304         struct nd_region *nd_region = to_nd_region(dev);
 305         unsigned long long available = 0;
 306
 307         /*
 308          * Flush in-flight updates and grab a snapshot of the available
 309          * size.  Of course, this value is potentially invalidated the
 310          * memory nvdimm_bus_lock() is dropped, but that's userspace's
 311          * problem to not race itself.
 312          */
 313         nvdimm_bus_lock(dev);
 314         wait_nvdimm_bus_probe_idle(dev);
 315         available = nd_region_available_dpa(nd_region);
 316         nvdimm_bus_unlock(dev);
 317
 318         return sprintf(buf, "%llu\n", available);
 319 }
 320 static DEVICE_ATTR_RO(available_size);
 321
 322 static ssize_t init_namespaces_show(struct device *dev,
 323                 struct device_attribute *attr, char *buf)
 324 {
 325         struct nd_region_data *ndrd = dev_get_drvdata(dev);
 326         ssize_t rc;
 327
 328         nvdimm_bus_lock(dev);
 329         if (ndrd)
 330                 rc = sprintf(buf, "%d/%d\n", ndrd->ns_active, ndrd->ns_count);
 331         else
 332                 rc = -ENXIO;
 333         nvdimm_bus_unlock(dev);
 334
 335         return rc;
 336 }
 337 static DEVICE_ATTR_RO(init_namespaces);
 338
 339 static ssize_t namespace_seed_show(struct device *dev,
 340                 struct device_attribute *attr, char *buf)
 341 {
 342         struct nd_region *nd_region = to_nd_region(dev);
 343         ssize_t rc;
 344
 345         nvdimm_bus_lock(dev);
 346         if (nd_region->ns_seed)
 347                 rc = sprintf(buf, "%s\n", dev_name(nd_region->ns_seed));
 348         else
 349                 rc = sprintf(buf, "\n");
 350         nvdimm_bus_unlock(dev);
 351         return rc;
 352 }
 353 static DEVICE_ATTR_RO(namespace_seed);
 354
 355 static ssize_t btt_seed_show(struct device *dev,
 356                 struct device_attribute *attr, char *buf)
 357 {
 358         struct nd_region *nd_region = to_nd_region(dev);
 359         ssize_t rc;
 360
 361         nvdimm_bus_lock(dev);
 362         if (nd_region->btt_seed)
 363                 rc = sprintf(buf, "%s\n", dev_name(nd_region->btt_seed));
 364         else
 365                 rc = sprintf(buf, "\n");
 366         nvdimm_bus_unlock(dev);
 367
 368         return rc;
 369 }
 370 static DEVICE_ATTR_RO(btt_seed);
 371
 372 static ssize_t pfn_seed_show(struct device *dev,
 373                 struct device_attribute *attr, char *buf)
 374 {
 375         struct nd_region *nd_region = to_nd_region(dev);
 376         ssize_t rc;
 377
 378         nvdimm_bus_lock(dev);
 379         if (nd_region->pfn_seed)
 380                 rc = sprintf(buf, "%s\n", dev_name(nd_region->pfn_seed));
 381         else
 382                 rc = sprintf(buf, "\n");
 383         nvdimm_bus_unlock(dev);
 384
 385         return rc;
 386 }
 387 static DEVICE_ATTR_RO(pfn_seed);
 388
 389 static ssize_t dax_seed_show(struct device *dev,
 390                 struct device_attribute *attr, char *buf)
 391 {
 392         struct nd_region *nd_region = to_nd_region(dev);
 393         ssize_t rc;
 394
 395         nvdimm_bus_lock(dev);
 396         if (nd_region->dax_seed)
 397                 rc = sprintf(buf, "%s\n", dev_name(nd_region->dax_seed));
 398         else
 399                 rc = sprintf(buf, "\n");
 400         nvdimm_bus_unlock(dev);
 401
 402         return rc;
 403 }
 404 static DEVICE_ATTR_RO(dax_seed);
 405
 406 static ssize_t read_only_show(struct device *dev,
 407                 struct device_attribute *attr, char *buf)
 408 {
 409         struct nd_region *nd_region = to_nd_region(dev);
 410
 411         return sprintf(buf, "%d\n", nd_region->ro);
 412 }
 413
 414 static ssize_t read_only_store(struct device *dev,
 415                 struct device_attribute *attr, const char *buf, size_t len)
 416 {
 417         bool ro;
 418         int rc = strtobool(buf, &ro);
 419         struct nd_region *nd_region = to_nd_region(dev);
 420
 421         if (rc)
 422                 return rc;
 423
 424         nd_region->ro = ro;
 425         return len;
 426 }
 427 static DEVICE_ATTR_RW(read_only);
 428
 429 static struct attribute *nd_region_attributes[] = {
 430         &dev_attr_size.attr,
 431         &dev_attr_nstype.attr,
 432         &dev_attr_mappings.attr,
 433         &dev_attr_btt_seed.attr,
 434         &dev_attr_pfn_seed.attr,
 435         &dev_attr_dax_seed.attr,
 436         &dev_attr_read_only.attr,
 437         &dev_attr_set_cookie.attr,
 438         &dev_attr_available_size.attr,
 439         &dev_attr_namespace_seed.attr,
 440         &dev_attr_init_namespaces.attr,
 441         NULL,
 442 };
 443
 444 static umode_t region_visible(struct kobject *kobj, struct attribute *a, int n)
 445 {
 446         struct device *dev = container_of(kobj, typeof(*dev), kobj);
 447         struct nd_region *nd_region = to_nd_region(dev);
 448         struct nd_interleave_set *nd_set = nd_region->nd_set;
 449         int type = nd_region_to_nstype(nd_region);
 450
 451         if (!is_nd_pmem(dev) && a == &dev_attr_pfn_seed.attr)
 452                 return 0;
 453
 454         if (!is_nd_pmem(dev) && a == &dev_attr_dax_seed.attr)
 455                 return 0;
 456
 457         if (a != &dev_attr_set_cookie.attr
 458                         && a != &dev_attr_available_size.attr)
 459                 return a->mode;
 460
 461         if ((type == ND_DEVICE_NAMESPACE_PMEM
 462                                 || type == ND_DEVICE_NAMESPACE_BLK)
 463                         && a == &dev_attr_available_size.attr)
 464                 return a->mode;
 465         else if (is_nd_pmem(dev) && nd_set)
 466                 return a->mode;
 467
 468         return 0;
 469 }
 470
 471 struct attribute_group nd_region_attribute_group = {
 472         .attrs = nd_region_attributes,
 473         .is_visible = region_visible,
 474 };
 475 EXPORT_SYMBOL_GPL(nd_region_attribute_group);
 476
 477 u64 nd_region_interleave_set_cookie(struct nd_region *nd_region)
 478 {
 479         struct nd_interleave_set *nd_set = nd_region->nd_set;
 480
 481         if (nd_set)
 482                 return nd_set->cookie;
 483         return 0;
 484 }
 485
 486 /*
 487  * Upon successful probe/remove, take/release a reference on the
 488  * associated interleave set (if present), and plant new btt + namespace
 489  * seeds.  Also, on the removal of a BLK region, notify the provider to
 490  * disable the region.
 491  */
 492 static void nd_region_notify_driver_action(struct nvdimm_bus *nvdimm_bus,
 493                 struct device *dev, bool probe)
 494 {
 495         struct nd_region *nd_region;
 496
 497         if (!probe && (is_nd_pmem(dev) || is_nd_blk(dev))) {
 498                 int i;
 499
 500                 nd_region = to_nd_region(dev);
 501                 for (i = 0; i < nd_region->ndr_mappings; i++) {
 502                         struct nd_mapping *nd_mapping = &nd_region->mapping[i];
 503                         struct nvdimm_drvdata *ndd = nd_mapping->ndd;
 504                         struct nvdimm *nvdimm = nd_mapping->nvdimm;
 505
 506                         kfree(nd_mapping->labels);
 507                         nd_mapping->labels = NULL;
 508                         put_ndd(ndd);
 509                         nd_mapping->ndd = NULL;
 510                         if (ndd)
 511                                 atomic_dec(&nvdimm->busy);
 512                 }
 513
 514                 if (is_nd_pmem(dev))
 515                         return;
 516         }
 517         if (dev->parent && is_nd_blk(dev->parent) && probe) {
 518                 nd_region = to_nd_region(dev->parent);
 519                 nvdimm_bus_lock(dev);
 520                 if (nd_region->ns_seed == dev)
 521                         nd_region_create_blk_seed(nd_region);
 522                 nvdimm_bus_unlock(dev);
 523         }
 524         if (is_nd_btt(dev) && probe) {
 525                 struct nd_btt *nd_btt = to_nd_btt(dev);
 526
 527                 nd_region = to_nd_region(dev->parent);
 528                 nvdimm_bus_lock(dev);
 529                 if (nd_region->btt_seed == dev)
 530                         nd_region_create_btt_seed(nd_region);
 531                 if (nd_region->ns_seed == &nd_btt->ndns->dev &&
 532                                 is_nd_blk(dev->parent))
 533                         nd_region_create_blk_seed(nd_region);
 534                 nvdimm_bus_unlock(dev);
 535         }
 536         if (is_nd_pfn(dev) && probe) {
 537                 nd_region = to_nd_region(dev->parent);
 538                 nvdimm_bus_lock(dev);
 539                 if (nd_region->pfn_seed == dev)
 540                         nd_region_create_pfn_seed(nd_region);
 541                 nvdimm_bus_unlock(dev);
 542         }
 543         if (is_nd_dax(dev) && probe) {
 544                 nd_region = to_nd_region(dev->parent);
 545                 nvdimm_bus_lock(dev);
 546                 if (nd_region->dax_seed == dev)
 547                         nd_region_create_dax_seed(nd_region);
 548                 nvdimm_bus_unlock(dev);
 549         }
 550 }
 551
 552 void nd_region_probe_success(struct nvdimm_bus *nvdimm_bus, struct device *dev)
 553 {
 554         nd_region_notify_driver_action(nvdimm_bus, dev, true);
 555 }
 556
 557 void nd_region_disable(struct nvdimm_bus *nvdimm_bus, struct device *dev)
 558 {
 559         nd_region_notify_driver_action(nvdimm_bus, dev, false);
 560 }
 561
 562 static ssize_t mappingN(struct device *dev, char *buf, int n)
 563 {
 564         struct nd_region *nd_region = to_nd_region(dev);
 565         struct nd_mapping *nd_mapping;
 566         struct nvdimm *nvdimm;
 567
 568         if (n >= nd_region->ndr_mappings)
 569                 return -ENXIO;
 570         nd_mapping = &nd_region->mapping[n];
 571         nvdimm = nd_mapping->nvdimm;
 572
 573         return sprintf(buf, "%s,%llu,%llu\n", dev_name(&nvdimm->dev),
 574                         nd_mapping->start, nd_mapping->size);
 575 }
 576
 577 #define REGION_MAPPING(idx) \
 578 static ssize_t mapping##idx##_show(struct device *dev,          \
 579                 struct device_attribute *attr, char *buf)       \
 580 {                                                               \
 581         return mappingN(dev, buf, idx);                         \
 582 }                                                               \
 583 static DEVICE_ATTR_RO(mapping##idx)
 584
 585 /*
 586  * 32 should be enough for a while, even in the presence of socket
 587  * interleave a 32-way interleave set is a degenerate case.
 588  */
 589 REGION_MAPPING(0);
 590 REGION_MAPPING(1);
 591 REGION_MAPPING(2);
 592 REGION_MAPPING(3);
 593 REGION_MAPPING(4);
 594 REGION_MAPPING(5);
 595 REGION_MAPPING(6);
 596 REGION_MAPPING(7);
 597 REGION_MAPPING(8);
 598 REGION_MAPPING(9);
 599 REGION_MAPPING(10);
 600 REGION_MAPPING(11);
 601 REGION_MAPPING(12);
 602 REGION_MAPPING(13);
 603 REGION_MAPPING(14);
 604 REGION_MAPPING(15);
 605 REGION_MAPPING(16);
 606 REGION_MAPPING(17);
 607 REGION_MAPPING(18);
 608 REGION_MAPPING(19);
 609 REGION_MAPPING(20);
 610 REGION_MAPPING(21);
 611 REGION_MAPPING(22);
 612 REGION_MAPPING(23);
 613 REGION_MAPPING(24);
 614 REGION_MAPPING(25);
 615 REGION_MAPPING(26);
 616 REGION_MAPPING(27);
 617 REGION_MAPPING(28);
 618 REGION_MAPPING(29);
 619 REGION_MAPPING(30);
 620 REGION_MAPPING(31);
 621
 622 static umode_t mapping_visible(struct kobject *kobj, struct attribute *a, int n)
 623 {
 624         struct device *dev = container_of(kobj, struct device, kobj);
 625         struct nd_region *nd_region = to_nd_region(dev);
 626
 627         if (n < nd_region->ndr_mappings)
 628                 return a->mode;
 629         return 0;
 630 }
 631
 632 static struct attribute *mapping_attributes[] = {
 633         &dev_attr_mapping0.attr,
 634         &dev_attr_mapping1.attr,
 635         &dev_attr_mapping2.attr,
 636         &dev_attr_mapping3.attr,
 637         &dev_attr_mapping4.attr,
 638         &dev_attr_mapping5.attr,
 639         &dev_attr_mapping6.attr,
 640         &dev_attr_mapping7.attr,
 641         &dev_attr_mapping8.attr,
 642         &dev_attr_mapping9.attr,
 643         &dev_attr_mapping10.attr,
 644         &dev_attr_mapping11.attr,
 645         &dev_attr_mapping12.attr,
 646         &dev_attr_mapping13.attr,
 647         &dev_attr_mapping14.attr,
 648         &dev_attr_mapping15.attr,
 649         &dev_attr_mapping16.attr,
 650         &dev_attr_mapping17.attr,
 651         &dev_attr_mapping18.attr,
 652         &dev_attr_mapping19.attr,
 653         &dev_attr_mapping20.attr,
 654         &dev_attr_mapping21.attr,
 655         &dev_attr_mapping22.attr,
 656         &dev_attr_mapping23.attr,
 657         &dev_attr_mapping24.attr,
 658         &dev_attr_mapping25.attr,
 659         &dev_attr_mapping26.attr,
 660         &dev_attr_mapping27.attr,
 661         &dev_attr_mapping28.attr,
 662         &dev_attr_mapping29.attr,
 663         &dev_attr_mapping30.attr,
 664         &dev_attr_mapping31.attr,
 665         NULL,
 666 };
 667
 668 struct attribute_group nd_mapping_attribute_group = {
 669         .is_visible = mapping_visible,
 670         .attrs = mapping_attributes,
 671 };
 672 EXPORT_SYMBOL_GPL(nd_mapping_attribute_group);
 673
 674 int nd_blk_region_init(struct nd_region *nd_region)
 675 {
 676         struct device *dev = &nd_region->dev;
 677         struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev);
 678
 679         if (!is_nd_blk(dev))
 680                 return 0;
 681
 682         if (nd_region->ndr_mappings < 1) {
 683                 dev_err(dev, "invalid BLK region\n");
 684                 return -ENXIO;
 685         }
 686
 687         return to_nd_blk_region(dev)->enable(nvdimm_bus, dev);
 688 }
 689
 690 /**
 691  * nd_region_acquire_lane - allocate and lock a lane
 692  * @nd_region: region id and number of lanes possible
 693  *
 694  * A lane correlates to a BLK-data-window and/or a log slot in the BTT.
 695  * We optimize for the common case where there are 256 lanes, one
 696  * per-cpu.  For larger systems we need to lock to share lanes.  For now
 697  * this implementation assumes the cost of maintaining an allocator for
 698  * free lanes is on the order of the lock hold time, so it implements a
 699  * static lane = cpu % num_lanes mapping.
 700  *
 701  * In the case of a BTT instance on top of a BLK namespace a lane may be
 702  * acquired recursively.  We lock on the first instance.
 703  *
 704  * In the case of a BTT instance on top of PMEM, we only acquire a lane
 705  * for the BTT metadata updates.
 706  */
 707 unsigned int nd_region_acquire_lane(struct nd_region *nd_region)
 708 {
 709         unsigned int cpu, lane;
 710
 711         cpu = get_cpu();
 712         if (nd_region->num_lanes < nr_cpu_ids) {
 713                 struct nd_percpu_lane *ndl_lock, *ndl_count;
 714
 715                 lane = cpu % nd_region->num_lanes;
 716                 ndl_count = per_cpu_ptr(nd_region->lane, cpu);
 717                 ndl_lock = per_cpu_ptr(nd_region->lane, lane);
 718                 if (ndl_count->count++ == 0)
 719                         spin_lock(&ndl_lock->lock);
 720         } else
 721                 lane = cpu;
 722
 723         return lane;
 724 }
 725 EXPORT_SYMBOL(nd_region_acquire_lane);
 726
 727 void nd_region_release_lane(struct nd_region *nd_region, unsigned int lane)
 728 {
 729         if (nd_region->num_lanes < nr_cpu_ids) {
 730                 unsigned int cpu = get_cpu();
 731                 struct nd_percpu_lane *ndl_lock, *ndl_count;
 732
 733                 ndl_count = per_cpu_ptr(nd_region->lane, cpu);
 734                 ndl_lock = per_cpu_ptr(nd_region->lane, lane);
 735                 if (--ndl_count->count == 0)
 736                         spin_unlock(&ndl_lock->lock);
 737                 put_cpu();
 738         }
 739         put_cpu();
 740 }
 741 EXPORT_SYMBOL(nd_region_release_lane);
 742
 743 static struct nd_region *nd_region_create(struct nvdimm_bus *nvdimm_bus,
 744                 struct nd_region_desc *ndr_desc, struct device_type *dev_type,
 745                 const char *caller)
 746 {
 747         struct nd_region *nd_region;
 748         struct device *dev;
 749         void *region_buf;
 750         unsigned int i;
 751         int ro = 0;
 752
 753         for (i = 0; i < ndr_desc->num_mappings; i++) {
 754                 struct nd_mapping *nd_mapping = &ndr_desc->nd_mapping[i];
 755                 struct nvdimm *nvdimm = nd_mapping->nvdimm;
 756
 757                 if ((nd_mapping->start | nd_mapping->size) % SZ_4K) {
 758                         dev_err(&nvdimm_bus->dev, "%s: %s mapping%d is not 4K aligned\n",
 759                                         caller, dev_name(&nvdimm->dev), i);
 760
 761                         return NULL;
 762                 }
 763
 764                 if (nvdimm->flags & NDD_UNARMED)
 765                         ro = 1;
 766         }
 767
 768         if (dev_type == &nd_blk_device_type) {
 769                 struct nd_blk_region_desc *ndbr_desc;
 770                 struct nd_blk_region *ndbr;
 771
 772                 ndbr_desc = to_blk_region_desc(ndr_desc);
 773                 ndbr = kzalloc(sizeof(*ndbr) + sizeof(struct nd_mapping)
 774                                 * ndr_desc->num_mappings,
 775                                 GFP_KERNEL);
 776                 if (ndbr) {
 777                         nd_region = &ndbr->nd_region;
 778                         ndbr->enable = ndbr_desc->enable;
 779                         ndbr->do_io = ndbr_desc->do_io;
 780                 }
 781                 region_buf = ndbr;
 782         } else {
 783                 nd_region = kzalloc(sizeof(struct nd_region)
 784                                 + sizeof(struct nd_mapping)
 785                                 * ndr_desc->num_mappings,
 786                                 GFP_KERNEL);
 787                 region_buf = nd_region;
 788         }
 789
 790         if (!region_buf)
 791                 return NULL;
 792         nd_region->id = ida_simple_get(&region_ida, 0, 0, GFP_KERNEL);
 793         if (nd_region->id < 0)
 794                 goto err_id;
 795
 796         nd_region->lane = alloc_percpu(struct nd_percpu_lane);
 797         if (!nd_region->lane)
 798                 goto err_percpu;
 799
 800         for (i = 0; i < nr_cpu_ids; i++) {
 801                 struct nd_percpu_lane *ndl;
 802
 803                 ndl = per_cpu_ptr(nd_region->lane, i);
 804                 spin_lock_init(&ndl->lock);
 805                 ndl->count = 0;
 806         }
 807
 808         memcpy(nd_region->mapping, ndr_desc->nd_mapping,
 809                         sizeof(struct nd_mapping) * ndr_desc->num_mappings);
 810         for (i = 0; i < ndr_desc->num_mappings; i++) {
 811                 struct nd_mapping *nd_mapping = &ndr_desc->nd_mapping[i];
 812                 struct nvdimm *nvdimm = nd_mapping->nvdimm;
 813
 814                 get_device(&nvdimm->dev);
 815         }
 816         nd_region->ndr_mappings = ndr_desc->num_mappings;
 817         nd_region->provider_data = ndr_desc->provider_data;
 818         nd_region->nd_set = ndr_desc->nd_set;
 819         nd_region->num_lanes = ndr_desc->num_lanes;
 820         nd_region->flags = ndr_desc->flags;
 821         nd_region->ro = ro;
 822         nd_region->numa_node = ndr_desc->numa_node;
 823         ida_init(&nd_region->ns_ida);
 824         ida_init(&nd_region->btt_ida);
 825         ida_init(&nd_region->pfn_ida);
 826         ida_init(&nd_region->dax_ida);
 827         dev = &nd_region->dev;
 828         dev_set_name(dev, "region%d", nd_region->id);
 829         dev->parent = &nvdimm_bus->dev;
 830         dev->type = dev_type;
 831         dev->groups = ndr_desc->attr_groups;
 832         nd_region->ndr_size = resource_size(ndr_desc->res);
 833         nd_region->ndr_start = ndr_desc->res->start;
 834         nd_device_register(dev);
 835
 836         return nd_region;
 837
 838  err_percpu:
 839         ida_simple_remove(&region_ida, nd_region->id);
 840  err_id:
 841         kfree(region_buf);
 842         return NULL;
 843 }
 844
 845 struct nd_region *nvdimm_pmem_region_create(struct nvdimm_bus *nvdimm_bus,
 846                 struct nd_region_desc *ndr_desc)
 847 {
 848         ndr_desc->num_lanes = ND_MAX_LANES;
 849         return nd_region_create(nvdimm_bus, ndr_desc, &nd_pmem_device_type,
 850                         __func__);
 851 }
 852 EXPORT_SYMBOL_GPL(nvdimm_pmem_region_create);
 853
 854 struct nd_region *nvdimm_blk_region_create(struct nvdimm_bus *nvdimm_bus,
 855                 struct nd_region_desc *ndr_desc)
 856 {
 857         if (ndr_desc->num_mappings > 1)
 858                 return NULL;
 859         ndr_desc->num_lanes = min(ndr_desc->num_lanes, ND_MAX_LANES);
 860         return nd_region_create(nvdimm_bus, ndr_desc, &nd_blk_device_type,
 861                         __func__);
 862 }
 863 EXPORT_SYMBOL_GPL(nvdimm_blk_region_create);
 864
 865 struct nd_region *nvdimm_volatile_region_create(struct nvdimm_bus *nvdimm_bus,
 866                 struct nd_region_desc *ndr_desc)
 867 {
 868         ndr_desc->num_lanes = ND_MAX_LANES;
 869         return nd_region_create(nvdimm_bus, ndr_desc, &nd_volatile_device_type,
 870                         __func__);
 871 }
 872 EXPORT_SYMBOL_GPL(nvdimm_volatile_region_create);
 873
 874 /**
 875  * nvdimm_flush - flush any posted write queues between the cpu and pmem media
 876  * @nd_region: blk or interleaved pmem region
 877  */
 878 void nvdimm_flush(struct nd_region *nd_region)
 879 {
 880         struct nd_region_data *ndrd = dev_get_drvdata(&nd_region->dev);
 881         int i;
 882
 883         /*
 884          * The first wmb() is needed to 'sfence' all previous writes
 885          * such that they are architecturally visible for the platform
 886          * buffer flush.  Note that we've already arranged for pmem
 887          * writes to avoid the cache via arch_memcpy_to_pmem().  The
 888          * final wmb() ensures ordering for the NVDIMM flush write.
 889          */
 890         wmb();
 891         for (i = 0; i < nd_region->ndr_mappings; i++)
 892                 if (ndrd->flush_wpq[i][0])
 893                         writeq(1, ndrd->flush_wpq[i][0]);
 894         wmb();
 895 }
 896 EXPORT_SYMBOL_GPL(nvdimm_flush);
 897
 898 /**
 899  * nvdimm_has_flush - determine write flushing requirements
 900  * @nd_region: blk or interleaved pmem region
 901  *
 902  * Returns 1 if writes require flushing
 903  * Returns 0 if writes do not require flushing
 904  * Returns -ENXIO if flushing capability can not be determined
 905  */
 906 int nvdimm_has_flush(struct nd_region *nd_region)
 907 {
 908         struct nd_region_data *ndrd = dev_get_drvdata(&nd_region->dev);
 909         int i;
 910
 911         /* no nvdimm == flushing capability unknown */
 912         if (nd_region->ndr_mappings == 0)
 913                 return -ENXIO;
 914
 915         for (i = 0; i < nd_region->ndr_mappings; i++)
 916                 /* flush hints present, flushing required */
 917                 if (ndrd->flush_wpq[i][0])
 918                         return 1;
 919
 920         /*
 921          * The platform defines dimm devices without hints, assume
 922          * platform persistence mechanism like ADR
 923          */
 924         return 0;
 925 }
 926 EXPORT_SYMBOL_GPL(nvdimm_has_flush);
 927
 928 void __exit nd_region_devs_exit(void)
 929 {
 930         ida_destroy(&region_ida);
 931 }