2 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
13 #include <linux/libnvdimm.h>
14 #include <linux/export.h>
15 #include <linux/module.h>
16 #include <linux/blkdev.h>
17 #include <linux/device.h>
18 #include <linux/ctype.h>
19 #include <linux/ndctl.h>
20 #include <linux/mutex.h>
21 #include <linux/slab.h>
25 LIST_HEAD(nvdimm_bus_list
);
26 DEFINE_MUTEX(nvdimm_bus_list_mutex
);
27 static DEFINE_IDA(nd_ida
);
29 void nvdimm_bus_lock(struct device
*dev
)
31 struct nvdimm_bus
*nvdimm_bus
= walk_to_nvdimm_bus(dev
);
35 mutex_lock(&nvdimm_bus
->reconfig_mutex
);
37 EXPORT_SYMBOL(nvdimm_bus_lock
);
39 void nvdimm_bus_unlock(struct device
*dev
)
41 struct nvdimm_bus
*nvdimm_bus
= walk_to_nvdimm_bus(dev
);
45 mutex_unlock(&nvdimm_bus
->reconfig_mutex
);
47 EXPORT_SYMBOL(nvdimm_bus_unlock
);
49 bool is_nvdimm_bus_locked(struct device
*dev
)
51 struct nvdimm_bus
*nvdimm_bus
= walk_to_nvdimm_bus(dev
);
55 return mutex_is_locked(&nvdimm_bus
->reconfig_mutex
);
57 EXPORT_SYMBOL(is_nvdimm_bus_locked
);
59 u64
nd_fletcher64(void *addr
, size_t len
, bool le
)
66 for (i
= 0; i
< len
/ sizeof(u32
); i
++) {
67 lo32
+= le
? le32_to_cpu((__le32
) buf
[i
]) : buf
[i
];
71 return hi32
<< 32 | lo32
;
73 EXPORT_SYMBOL_GPL(nd_fletcher64
);
75 static void nvdimm_bus_release(struct device
*dev
)
77 struct nvdimm_bus
*nvdimm_bus
;
79 nvdimm_bus
= container_of(dev
, struct nvdimm_bus
, dev
);
80 ida_simple_remove(&nd_ida
, nvdimm_bus
->id
);
84 struct nvdimm_bus
*to_nvdimm_bus(struct device
*dev
)
86 struct nvdimm_bus
*nvdimm_bus
;
88 nvdimm_bus
= container_of(dev
, struct nvdimm_bus
, dev
);
89 WARN_ON(nvdimm_bus
->dev
.release
!= nvdimm_bus_release
);
92 EXPORT_SYMBOL_GPL(to_nvdimm_bus
);
94 struct nvdimm_bus_descriptor
*to_nd_desc(struct nvdimm_bus
*nvdimm_bus
)
96 /* struct nvdimm_bus definition is private to libnvdimm */
97 return nvdimm_bus
->nd_desc
;
99 EXPORT_SYMBOL_GPL(to_nd_desc
);
101 struct nvdimm_bus
*walk_to_nvdimm_bus(struct device
*nd_dev
)
105 for (dev
= nd_dev
; dev
; dev
= dev
->parent
)
106 if (dev
->release
== nvdimm_bus_release
)
108 dev_WARN_ONCE(nd_dev
, !dev
, "invalid dev, not on nd bus\n");
110 return to_nvdimm_bus(dev
);
114 static bool is_uuid_sep(char sep
)
116 if (sep
== '\n' || sep
== '-' || sep
== ':' || sep
== '\0')
121 static int nd_uuid_parse(struct device
*dev
, u8
*uuid_out
, const char *buf
,
124 const char *str
= buf
;
128 for (i
= 0; i
< 16; i
++) {
129 if (!isxdigit(str
[0]) || !isxdigit(str
[1])) {
130 dev_dbg(dev
, "%s: pos: %d buf[%zd]: %c buf[%zd]: %c\n",
131 __func__
, i
, str
- buf
, str
[0],
132 str
+ 1 - buf
, str
[1]);
136 uuid
[i
] = (hex_to_bin(str
[0]) << 4) | hex_to_bin(str
[1]);
138 if (is_uuid_sep(*str
))
142 memcpy(uuid_out
, uuid
, sizeof(uuid
));
147 * nd_uuid_store: common implementation for writing 'uuid' sysfs attributes
148 * @dev: container device for the uuid property
149 * @uuid_out: uuid buffer to replace
150 * @buf: raw sysfs buffer to parse
152 * Enforce that uuids can only be changed while the device is disabled
154 * LOCKING: expects device_lock() is held on entry
156 int nd_uuid_store(struct device
*dev
, u8
**uuid_out
, const char *buf
,
165 rc
= nd_uuid_parse(dev
, uuid
, buf
, len
);
170 *uuid_out
= kmemdup(uuid
, sizeof(uuid
), GFP_KERNEL
);
177 ssize_t
nd_sector_size_show(unsigned long current_lbasize
,
178 const unsigned long *supported
, char *buf
)
183 for (i
= 0; supported
[i
]; i
++)
184 if (current_lbasize
== supported
[i
])
185 len
+= sprintf(buf
+ len
, "[%ld] ", supported
[i
]);
187 len
+= sprintf(buf
+ len
, "%ld ", supported
[i
]);
188 len
+= sprintf(buf
+ len
, "\n");
192 ssize_t
nd_sector_size_store(struct device
*dev
, const char *buf
,
193 unsigned long *current_lbasize
, const unsigned long *supported
)
195 unsigned long lbasize
;
201 rc
= kstrtoul(buf
, 0, &lbasize
);
205 for (i
= 0; supported
[i
]; i
++)
206 if (lbasize
== supported
[i
])
210 *current_lbasize
= lbasize
;
217 void __nd_iostat_start(struct bio
*bio
, unsigned long *start
)
219 struct gendisk
*disk
= bio
->bi_bdev
->bd_disk
;
220 const int rw
= bio_data_dir(bio
);
221 int cpu
= part_stat_lock();
224 part_round_stats(cpu
, &disk
->part0
);
225 part_stat_inc(cpu
, &disk
->part0
, ios
[rw
]);
226 part_stat_add(cpu
, &disk
->part0
, sectors
[rw
], bio_sectors(bio
));
227 part_inc_in_flight(&disk
->part0
, rw
);
230 EXPORT_SYMBOL(__nd_iostat_start
);
232 void nd_iostat_end(struct bio
*bio
, unsigned long start
)
234 struct gendisk
*disk
= bio
->bi_bdev
->bd_disk
;
235 unsigned long duration
= jiffies
- start
;
236 const int rw
= bio_data_dir(bio
);
237 int cpu
= part_stat_lock();
239 part_stat_add(cpu
, &disk
->part0
, ticks
[rw
], duration
);
240 part_round_stats(cpu
, &disk
->part0
);
241 part_dec_in_flight(&disk
->part0
, rw
);
244 EXPORT_SYMBOL(nd_iostat_end
);
246 static ssize_t
commands_show(struct device
*dev
,
247 struct device_attribute
*attr
, char *buf
)
250 struct nvdimm_bus
*nvdimm_bus
= to_nvdimm_bus(dev
);
251 struct nvdimm_bus_descriptor
*nd_desc
= nvdimm_bus
->nd_desc
;
253 for_each_set_bit(cmd
, &nd_desc
->dsm_mask
, BITS_PER_LONG
)
254 len
+= sprintf(buf
+ len
, "%s ", nvdimm_bus_cmd_name(cmd
));
255 len
+= sprintf(buf
+ len
, "\n");
258 static DEVICE_ATTR_RO(commands
);
260 static const char *nvdimm_bus_provider(struct nvdimm_bus
*nvdimm_bus
)
262 struct nvdimm_bus_descriptor
*nd_desc
= nvdimm_bus
->nd_desc
;
263 struct device
*parent
= nvdimm_bus
->dev
.parent
;
265 if (nd_desc
->provider_name
)
266 return nd_desc
->provider_name
;
268 return dev_name(parent
);
273 static ssize_t
provider_show(struct device
*dev
,
274 struct device_attribute
*attr
, char *buf
)
276 struct nvdimm_bus
*nvdimm_bus
= to_nvdimm_bus(dev
);
278 return sprintf(buf
, "%s\n", nvdimm_bus_provider(nvdimm_bus
));
280 static DEVICE_ATTR_RO(provider
);
282 static int flush_namespaces(struct device
*dev
, void *data
)
289 static int flush_regions_dimms(struct device
*dev
, void *data
)
293 device_for_each_child(dev
, NULL
, flush_namespaces
);
297 static ssize_t
wait_probe_show(struct device
*dev
,
298 struct device_attribute
*attr
, char *buf
)
301 device_for_each_child(dev
, NULL
, flush_regions_dimms
);
302 return sprintf(buf
, "1\n");
304 static DEVICE_ATTR_RO(wait_probe
);
306 static struct attribute
*nvdimm_bus_attributes
[] = {
307 &dev_attr_commands
.attr
,
308 &dev_attr_wait_probe
.attr
,
309 &dev_attr_provider
.attr
,
313 struct attribute_group nvdimm_bus_attribute_group
= {
314 .attrs
= nvdimm_bus_attributes
,
316 EXPORT_SYMBOL_GPL(nvdimm_bus_attribute_group
);
318 struct nvdimm_bus
*__nvdimm_bus_register(struct device
*parent
,
319 struct nvdimm_bus_descriptor
*nd_desc
, struct module
*module
)
321 struct nvdimm_bus
*nvdimm_bus
;
324 nvdimm_bus
= kzalloc(sizeof(*nvdimm_bus
), GFP_KERNEL
);
327 INIT_LIST_HEAD(&nvdimm_bus
->list
);
328 INIT_LIST_HEAD(&nvdimm_bus
->poison_list
);
329 init_waitqueue_head(&nvdimm_bus
->probe_wait
);
330 nvdimm_bus
->id
= ida_simple_get(&nd_ida
, 0, 0, GFP_KERNEL
);
331 mutex_init(&nvdimm_bus
->reconfig_mutex
);
332 if (nvdimm_bus
->id
< 0) {
336 nvdimm_bus
->nd_desc
= nd_desc
;
337 nvdimm_bus
->module
= module
;
338 nvdimm_bus
->dev
.parent
= parent
;
339 nvdimm_bus
->dev
.release
= nvdimm_bus_release
;
340 nvdimm_bus
->dev
.groups
= nd_desc
->attr_groups
;
341 dev_set_name(&nvdimm_bus
->dev
, "ndbus%d", nvdimm_bus
->id
);
342 rc
= device_register(&nvdimm_bus
->dev
);
344 dev_dbg(&nvdimm_bus
->dev
, "registration failed: %d\n", rc
);
348 rc
= nvdimm_bus_create_ndctl(nvdimm_bus
);
352 mutex_lock(&nvdimm_bus_list_mutex
);
353 list_add_tail(&nvdimm_bus
->list
, &nvdimm_bus_list
);
354 mutex_unlock(&nvdimm_bus_list_mutex
);
358 put_device(&nvdimm_bus
->dev
);
361 EXPORT_SYMBOL_GPL(__nvdimm_bus_register
);
364 * __add_badblock_range() - Convert a physical address range to bad sectors
365 * @disk: the disk associated with the namespace
366 * @ns_offset: namespace offset where the error range begins (in bytes)
367 * @len: number of bytes of poison to be added
369 * This assumes that the range provided with (ns_offset, len) is within
370 * the bounds of physical addresses for this namespace, i.e. lies in the
371 * interval [ns_start, ns_start + ns_size)
373 static int __add_badblock_range(struct gendisk
*disk
, u64 ns_offset
, u64 len
)
375 unsigned int sector_size
= queue_logical_block_size(disk
->queue
);
376 sector_t start_sector
;
381 start_sector
= div_u64(ns_offset
, sector_size
);
382 num_sectors
= div_u64_rem(len
, sector_size
, &rem
);
387 rc
= disk_alloc_badblocks(disk
);
392 if (unlikely(num_sectors
> (u64
)INT_MAX
)) {
393 u64 remaining
= num_sectors
;
394 sector_t s
= start_sector
;
397 int done
= min_t(u64
, remaining
, INT_MAX
);
399 rc
= disk_set_badblocks(disk
, s
, done
);
407 return disk_set_badblocks(disk
, start_sector
, num_sectors
);
411 * nvdimm_namespace_add_poison() - Convert a list of poison ranges to badblocks
412 * @disk: the gendisk associated with the namespace where badblocks
414 * @offset: offset at the start of the namespace before 'sector 0'
415 * @ndns: the namespace containing poison ranges
417 * The poison list generated during NFIT initialization may contain multiple,
418 * possibly overlapping ranges in the SPA (System Physical Address) space.
419 * Compare each of these ranges to the namespace currently being initialized,
420 * and add badblocks to the gendisk for all matching sub-ranges
425 int nvdimm_namespace_add_poison(struct gendisk
*disk
, resource_size_t offset
,
426 struct nd_namespace_common
*ndns
)
428 struct nd_namespace_io
*nsio
= to_nd_namespace_io(&ndns
->dev
);
429 struct nd_region
*nd_region
= to_nd_region(ndns
->dev
.parent
);
430 struct nvdimm_bus
*nvdimm_bus
;
431 struct list_head
*poison_list
;
432 u64 ns_start
, ns_end
, ns_size
;
433 struct nd_poison
*pl
;
436 ns_size
= nvdimm_namespace_capacity(ndns
) - offset
;
437 ns_start
= nsio
->res
.start
+ offset
;
438 ns_end
= nsio
->res
.end
;
440 nvdimm_bus
= to_nvdimm_bus(nd_region
->dev
.parent
);
441 poison_list
= &nvdimm_bus
->poison_list
;
442 if (list_empty(poison_list
))
445 list_for_each_entry(pl
, poison_list
, list
) {
446 u64 pl_end
= pl
->start
+ pl
->length
- 1;
448 /* Discard intervals with no intersection */
449 if (pl_end
< ns_start
)
451 if (pl
->start
> ns_end
)
453 /* Deal with any overlap after start of the namespace */
454 if (pl
->start
>= ns_start
) {
455 u64 start
= pl
->start
;
458 if (pl_end
<= ns_end
)
461 len
= ns_start
+ ns_size
- pl
->start
;
463 rc
= __add_badblock_range(disk
, start
- ns_start
, len
);
466 dev_info(&nvdimm_bus
->dev
,
467 "Found a poison range (0x%llx, 0x%llx)\n",
471 /* Deal with overlap for poison starting before the namespace */
472 if (pl
->start
< ns_start
) {
476 len
= pl
->start
+ pl
->length
- ns_start
;
480 rc
= __add_badblock_range(disk
, 0, len
);
483 dev_info(&nvdimm_bus
->dev
,
484 "Found a poison range (0x%llx, 0x%llx)\n",
491 EXPORT_SYMBOL_GPL(nvdimm_namespace_add_poison
);
493 static int __add_poison(struct nvdimm_bus
*nvdimm_bus
, u64 addr
, u64 length
)
495 struct nd_poison
*pl
;
497 pl
= kzalloc(sizeof(*pl
), GFP_KERNEL
);
503 list_add_tail(&pl
->list
, &nvdimm_bus
->poison_list
);
508 int nvdimm_bus_add_poison(struct nvdimm_bus
*nvdimm_bus
, u64 addr
, u64 length
)
510 struct nd_poison
*pl
;
512 if (list_empty(&nvdimm_bus
->poison_list
))
513 return __add_poison(nvdimm_bus
, addr
, length
);
516 * There is a chance this is a duplicate, check for those first.
517 * This will be the common case as ARS_STATUS returns all known
518 * errors in the SPA space, and we can't query it per region
520 list_for_each_entry(pl
, &nvdimm_bus
->poison_list
, list
)
521 if (pl
->start
== addr
) {
522 /* If length has changed, update this list entry */
523 if (pl
->length
!= length
)
529 * If not a duplicate or a simple length update, add the entry as is,
530 * as any overlapping ranges will get resolved when the list is consumed
531 * and converted to badblocks
533 return __add_poison(nvdimm_bus
, addr
, length
);
535 EXPORT_SYMBOL_GPL(nvdimm_bus_add_poison
);
537 static void free_poison_list(struct list_head
*poison_list
)
539 struct nd_poison
*pl
, *next
;
541 list_for_each_entry_safe(pl
, next
, poison_list
, list
) {
545 list_del_init(poison_list
);
548 static int child_unregister(struct device
*dev
, void *data
)
551 * the singular ndctl class device per bus needs to be
552 * "device_destroy"ed, so skip it here
554 * i.e. remove classless children
559 nd_device_unregister(dev
, ND_SYNC
);
563 void nvdimm_bus_unregister(struct nvdimm_bus
*nvdimm_bus
)
568 mutex_lock(&nvdimm_bus_list_mutex
);
569 list_del_init(&nvdimm_bus
->list
);
570 mutex_unlock(&nvdimm_bus_list_mutex
);
573 device_for_each_child(&nvdimm_bus
->dev
, NULL
, child_unregister
);
574 free_poison_list(&nvdimm_bus
->poison_list
);
575 nvdimm_bus_destroy_ndctl(nvdimm_bus
);
577 device_unregister(&nvdimm_bus
->dev
);
579 EXPORT_SYMBOL_GPL(nvdimm_bus_unregister
);
581 #ifdef CONFIG_BLK_DEV_INTEGRITY
582 int nd_integrity_init(struct gendisk
*disk
, unsigned long meta_size
)
584 struct blk_integrity bi
;
590 bi
.tuple_size
= meta_size
;
591 bi
.tag_size
= meta_size
;
593 blk_integrity_register(disk
, &bi
);
594 blk_queue_max_integrity_segments(disk
->queue
, 1);
598 EXPORT_SYMBOL(nd_integrity_init
);
600 #else /* CONFIG_BLK_DEV_INTEGRITY */
601 int nd_integrity_init(struct gendisk
*disk
, unsigned long meta_size
)
605 EXPORT_SYMBOL(nd_integrity_init
);
609 static __init
int libnvdimm_init(void)
613 rc
= nvdimm_bus_init();
619 rc
= nd_region_init();
630 static __exit
void libnvdimm_exit(void)
632 WARN_ON(!list_empty(&nvdimm_bus_list
));
638 MODULE_LICENSE("GPL v2");
639 MODULE_AUTHOR("Intel Corporation");
640 subsys_initcall(libnvdimm_init
);
641 module_exit(libnvdimm_exit
);