2 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License as
6 * published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
13 #include <linux/libnvdimm.h>
14 #include <linux/badblocks.h>
15 #include <linux/export.h>
16 #include <linux/module.h>
17 #include <linux/blkdev.h>
18 #include <linux/device.h>
19 #include <linux/ctype.h>
20 #include <linux/ndctl.h>
21 #include <linux/mutex.h>
22 #include <linux/slab.h>
26 LIST_HEAD(nvdimm_bus_list
);
27 DEFINE_MUTEX(nvdimm_bus_list_mutex
);
28 static DEFINE_IDA(nd_ida
);
30 void nvdimm_bus_lock(struct device
*dev
)
32 struct nvdimm_bus
*nvdimm_bus
= walk_to_nvdimm_bus(dev
);
36 mutex_lock(&nvdimm_bus
->reconfig_mutex
);
38 EXPORT_SYMBOL(nvdimm_bus_lock
);
40 void nvdimm_bus_unlock(struct device
*dev
)
42 struct nvdimm_bus
*nvdimm_bus
= walk_to_nvdimm_bus(dev
);
46 mutex_unlock(&nvdimm_bus
->reconfig_mutex
);
48 EXPORT_SYMBOL(nvdimm_bus_unlock
);
50 bool is_nvdimm_bus_locked(struct device
*dev
)
52 struct nvdimm_bus
*nvdimm_bus
= walk_to_nvdimm_bus(dev
);
56 return mutex_is_locked(&nvdimm_bus
->reconfig_mutex
);
58 EXPORT_SYMBOL(is_nvdimm_bus_locked
);
60 u64
nd_fletcher64(void *addr
, size_t len
, bool le
)
67 for (i
= 0; i
< len
/ sizeof(u32
); i
++) {
68 lo32
+= le
? le32_to_cpu((__le32
) buf
[i
]) : buf
[i
];
72 return hi32
<< 32 | lo32
;
74 EXPORT_SYMBOL_GPL(nd_fletcher64
);
76 static void nvdimm_bus_release(struct device
*dev
)
78 struct nvdimm_bus
*nvdimm_bus
;
80 nvdimm_bus
= container_of(dev
, struct nvdimm_bus
, dev
);
81 ida_simple_remove(&nd_ida
, nvdimm_bus
->id
);
85 struct nvdimm_bus
*to_nvdimm_bus(struct device
*dev
)
87 struct nvdimm_bus
*nvdimm_bus
;
89 nvdimm_bus
= container_of(dev
, struct nvdimm_bus
, dev
);
90 WARN_ON(nvdimm_bus
->dev
.release
!= nvdimm_bus_release
);
93 EXPORT_SYMBOL_GPL(to_nvdimm_bus
);
95 struct nvdimm_bus_descriptor
*to_nd_desc(struct nvdimm_bus
*nvdimm_bus
)
97 /* struct nvdimm_bus definition is private to libnvdimm */
98 return nvdimm_bus
->nd_desc
;
100 EXPORT_SYMBOL_GPL(to_nd_desc
);
102 struct nvdimm_bus
*walk_to_nvdimm_bus(struct device
*nd_dev
)
106 for (dev
= nd_dev
; dev
; dev
= dev
->parent
)
107 if (dev
->release
== nvdimm_bus_release
)
109 dev_WARN_ONCE(nd_dev
, !dev
, "invalid dev, not on nd bus\n");
111 return to_nvdimm_bus(dev
);
115 static bool is_uuid_sep(char sep
)
117 if (sep
== '\n' || sep
== '-' || sep
== ':' || sep
== '\0')
122 static int nd_uuid_parse(struct device
*dev
, u8
*uuid_out
, const char *buf
,
125 const char *str
= buf
;
129 for (i
= 0; i
< 16; i
++) {
130 if (!isxdigit(str
[0]) || !isxdigit(str
[1])) {
131 dev_dbg(dev
, "%s: pos: %d buf[%zd]: %c buf[%zd]: %c\n",
132 __func__
, i
, str
- buf
, str
[0],
133 str
+ 1 - buf
, str
[1]);
137 uuid
[i
] = (hex_to_bin(str
[0]) << 4) | hex_to_bin(str
[1]);
139 if (is_uuid_sep(*str
))
143 memcpy(uuid_out
, uuid
, sizeof(uuid
));
148 * nd_uuid_store: common implementation for writing 'uuid' sysfs attributes
149 * @dev: container device for the uuid property
150 * @uuid_out: uuid buffer to replace
151 * @buf: raw sysfs buffer to parse
153 * Enforce that uuids can only be changed while the device is disabled
155 * LOCKING: expects device_lock() is held on entry
157 int nd_uuid_store(struct device
*dev
, u8
**uuid_out
, const char *buf
,
166 rc
= nd_uuid_parse(dev
, uuid
, buf
, len
);
171 *uuid_out
= kmemdup(uuid
, sizeof(uuid
), GFP_KERNEL
);
178 ssize_t
nd_sector_size_show(unsigned long current_lbasize
,
179 const unsigned long *supported
, char *buf
)
184 for (i
= 0; supported
[i
]; i
++)
185 if (current_lbasize
== supported
[i
])
186 len
+= sprintf(buf
+ len
, "[%ld] ", supported
[i
]);
188 len
+= sprintf(buf
+ len
, "%ld ", supported
[i
]);
189 len
+= sprintf(buf
+ len
, "\n");
193 ssize_t
nd_sector_size_store(struct device
*dev
, const char *buf
,
194 unsigned long *current_lbasize
, const unsigned long *supported
)
196 unsigned long lbasize
;
202 rc
= kstrtoul(buf
, 0, &lbasize
);
206 for (i
= 0; supported
[i
]; i
++)
207 if (lbasize
== supported
[i
])
211 *current_lbasize
= lbasize
;
218 void __nd_iostat_start(struct bio
*bio
, unsigned long *start
)
220 struct gendisk
*disk
= bio
->bi_bdev
->bd_disk
;
221 const int rw
= bio_data_dir(bio
);
222 int cpu
= part_stat_lock();
225 part_round_stats(cpu
, &disk
->part0
);
226 part_stat_inc(cpu
, &disk
->part0
, ios
[rw
]);
227 part_stat_add(cpu
, &disk
->part0
, sectors
[rw
], bio_sectors(bio
));
228 part_inc_in_flight(&disk
->part0
, rw
);
231 EXPORT_SYMBOL(__nd_iostat_start
);
233 void nd_iostat_end(struct bio
*bio
, unsigned long start
)
235 struct gendisk
*disk
= bio
->bi_bdev
->bd_disk
;
236 unsigned long duration
= jiffies
- start
;
237 const int rw
= bio_data_dir(bio
);
238 int cpu
= part_stat_lock();
240 part_stat_add(cpu
, &disk
->part0
, ticks
[rw
], duration
);
241 part_round_stats(cpu
, &disk
->part0
);
242 part_dec_in_flight(&disk
->part0
, rw
);
245 EXPORT_SYMBOL(nd_iostat_end
);
247 static ssize_t
commands_show(struct device
*dev
,
248 struct device_attribute
*attr
, char *buf
)
251 struct nvdimm_bus
*nvdimm_bus
= to_nvdimm_bus(dev
);
252 struct nvdimm_bus_descriptor
*nd_desc
= nvdimm_bus
->nd_desc
;
254 for_each_set_bit(cmd
, &nd_desc
->dsm_mask
, BITS_PER_LONG
)
255 len
+= sprintf(buf
+ len
, "%s ", nvdimm_bus_cmd_name(cmd
));
256 len
+= sprintf(buf
+ len
, "\n");
259 static DEVICE_ATTR_RO(commands
);
261 static const char *nvdimm_bus_provider(struct nvdimm_bus
*nvdimm_bus
)
263 struct nvdimm_bus_descriptor
*nd_desc
= nvdimm_bus
->nd_desc
;
264 struct device
*parent
= nvdimm_bus
->dev
.parent
;
266 if (nd_desc
->provider_name
)
267 return nd_desc
->provider_name
;
269 return dev_name(parent
);
274 static ssize_t
provider_show(struct device
*dev
,
275 struct device_attribute
*attr
, char *buf
)
277 struct nvdimm_bus
*nvdimm_bus
= to_nvdimm_bus(dev
);
279 return sprintf(buf
, "%s\n", nvdimm_bus_provider(nvdimm_bus
));
281 static DEVICE_ATTR_RO(provider
);
283 static int flush_namespaces(struct device
*dev
, void *data
)
290 static int flush_regions_dimms(struct device
*dev
, void *data
)
294 device_for_each_child(dev
, NULL
, flush_namespaces
);
298 static ssize_t
wait_probe_show(struct device
*dev
,
299 struct device_attribute
*attr
, char *buf
)
302 device_for_each_child(dev
, NULL
, flush_regions_dimms
);
303 return sprintf(buf
, "1\n");
305 static DEVICE_ATTR_RO(wait_probe
);
307 static struct attribute
*nvdimm_bus_attributes
[] = {
308 &dev_attr_commands
.attr
,
309 &dev_attr_wait_probe
.attr
,
310 &dev_attr_provider
.attr
,
314 struct attribute_group nvdimm_bus_attribute_group
= {
315 .attrs
= nvdimm_bus_attributes
,
317 EXPORT_SYMBOL_GPL(nvdimm_bus_attribute_group
);
319 struct nvdimm_bus
*__nvdimm_bus_register(struct device
*parent
,
320 struct nvdimm_bus_descriptor
*nd_desc
, struct module
*module
)
322 struct nvdimm_bus
*nvdimm_bus
;
325 nvdimm_bus
= kzalloc(sizeof(*nvdimm_bus
), GFP_KERNEL
);
328 INIT_LIST_HEAD(&nvdimm_bus
->list
);
329 INIT_LIST_HEAD(&nvdimm_bus
->poison_list
);
330 init_waitqueue_head(&nvdimm_bus
->probe_wait
);
331 nvdimm_bus
->id
= ida_simple_get(&nd_ida
, 0, 0, GFP_KERNEL
);
332 mutex_init(&nvdimm_bus
->reconfig_mutex
);
333 if (nvdimm_bus
->id
< 0) {
337 nvdimm_bus
->nd_desc
= nd_desc
;
338 nvdimm_bus
->module
= module
;
339 nvdimm_bus
->dev
.parent
= parent
;
340 nvdimm_bus
->dev
.release
= nvdimm_bus_release
;
341 nvdimm_bus
->dev
.groups
= nd_desc
->attr_groups
;
342 dev_set_name(&nvdimm_bus
->dev
, "ndbus%d", nvdimm_bus
->id
);
343 rc
= device_register(&nvdimm_bus
->dev
);
345 dev_dbg(&nvdimm_bus
->dev
, "registration failed: %d\n", rc
);
349 rc
= nvdimm_bus_create_ndctl(nvdimm_bus
);
353 mutex_lock(&nvdimm_bus_list_mutex
);
354 list_add_tail(&nvdimm_bus
->list
, &nvdimm_bus_list
);
355 mutex_unlock(&nvdimm_bus_list_mutex
);
359 put_device(&nvdimm_bus
->dev
);
362 EXPORT_SYMBOL_GPL(__nvdimm_bus_register
);
364 static void set_badblock(struct badblocks
*bb
, sector_t s
, int num
)
366 dev_dbg(bb
->dev
, "Found a poison range (0x%llx, 0x%llx)\n",
367 (u64
) s
* 512, (u64
) num
* 512);
368 /* this isn't an error as the hardware will still throw an exception */
369 if (badblocks_set(bb
, s
, num
, 1))
370 dev_info_once(bb
->dev
, "%s: failed for sector %llx\n",
375 * __add_badblock_range() - Convert a physical address range to bad sectors
376 * @bb: badblocks instance to populate
377 * @ns_offset: namespace offset where the error range begins (in bytes)
378 * @len: number of bytes of poison to be added
380 * This assumes that the range provided with (ns_offset, len) is within
381 * the bounds of physical addresses for this namespace, i.e. lies in the
382 * interval [ns_start, ns_start + ns_size)
384 static void __add_badblock_range(struct badblocks
*bb
, u64 ns_offset
, u64 len
)
386 const unsigned int sector_size
= 512;
387 sector_t start_sector
;
391 start_sector
= div_u64(ns_offset
, sector_size
);
392 num_sectors
= div_u64_rem(len
, sector_size
, &rem
);
396 if (unlikely(num_sectors
> (u64
)INT_MAX
)) {
397 u64 remaining
= num_sectors
;
398 sector_t s
= start_sector
;
401 int done
= min_t(u64
, remaining
, INT_MAX
);
403 set_badblock(bb
, s
, done
);
408 set_badblock(bb
, start_sector
, num_sectors
);
412 * nvdimm_namespace_add_poison() - Convert a list of poison ranges to badblocks
413 * @ndns: the namespace containing poison ranges
414 * @bb: badblocks instance to populate
415 * @offset: offset at the start of the namespace before 'sector 0'
417 * The poison list generated during NFIT initialization may contain multiple,
418 * possibly overlapping ranges in the SPA (System Physical Address) space.
419 * Compare each of these ranges to the namespace currently being initialized,
420 * and add badblocks to the gendisk for all matching sub-ranges
422 void nvdimm_namespace_add_poison(struct nd_namespace_common
*ndns
,
423 struct badblocks
*bb
, resource_size_t offset
)
425 struct nd_namespace_io
*nsio
= to_nd_namespace_io(&ndns
->dev
);
426 struct nd_region
*nd_region
= to_nd_region(ndns
->dev
.parent
);
427 struct nvdimm_bus
*nvdimm_bus
;
428 struct list_head
*poison_list
;
429 u64 ns_start
, ns_end
, ns_size
;
430 struct nd_poison
*pl
;
432 ns_size
= nvdimm_namespace_capacity(ndns
) - offset
;
433 ns_start
= nsio
->res
.start
+ offset
;
434 ns_end
= nsio
->res
.end
;
436 nvdimm_bus
= to_nvdimm_bus(nd_region
->dev
.parent
);
437 poison_list
= &nvdimm_bus
->poison_list
;
438 if (list_empty(poison_list
))
441 list_for_each_entry(pl
, poison_list
, list
) {
442 u64 pl_end
= pl
->start
+ pl
->length
- 1;
444 /* Discard intervals with no intersection */
445 if (pl_end
< ns_start
)
447 if (pl
->start
> ns_end
)
449 /* Deal with any overlap after start of the namespace */
450 if (pl
->start
>= ns_start
) {
451 u64 start
= pl
->start
;
454 if (pl_end
<= ns_end
)
457 len
= ns_start
+ ns_size
- pl
->start
;
458 __add_badblock_range(bb
, start
- ns_start
, len
);
461 /* Deal with overlap for poison starting before the namespace */
462 if (pl
->start
< ns_start
) {
466 len
= pl
->start
+ pl
->length
- ns_start
;
469 __add_badblock_range(bb
, 0, len
);
473 EXPORT_SYMBOL_GPL(nvdimm_namespace_add_poison
);
475 static int __add_poison(struct nvdimm_bus
*nvdimm_bus
, u64 addr
, u64 length
)
477 struct nd_poison
*pl
;
479 pl
= kzalloc(sizeof(*pl
), GFP_KERNEL
);
485 list_add_tail(&pl
->list
, &nvdimm_bus
->poison_list
);
490 int nvdimm_bus_add_poison(struct nvdimm_bus
*nvdimm_bus
, u64 addr
, u64 length
)
492 struct nd_poison
*pl
;
494 if (list_empty(&nvdimm_bus
->poison_list
))
495 return __add_poison(nvdimm_bus
, addr
, length
);
498 * There is a chance this is a duplicate, check for those first.
499 * This will be the common case as ARS_STATUS returns all known
500 * errors in the SPA space, and we can't query it per region
502 list_for_each_entry(pl
, &nvdimm_bus
->poison_list
, list
)
503 if (pl
->start
== addr
) {
504 /* If length has changed, update this list entry */
505 if (pl
->length
!= length
)
511 * If not a duplicate or a simple length update, add the entry as is,
512 * as any overlapping ranges will get resolved when the list is consumed
513 * and converted to badblocks
515 return __add_poison(nvdimm_bus
, addr
, length
);
517 EXPORT_SYMBOL_GPL(nvdimm_bus_add_poison
);
519 static void free_poison_list(struct list_head
*poison_list
)
521 struct nd_poison
*pl
, *next
;
523 list_for_each_entry_safe(pl
, next
, poison_list
, list
) {
527 list_del_init(poison_list
);
530 static int child_unregister(struct device
*dev
, void *data
)
533 * the singular ndctl class device per bus needs to be
534 * "device_destroy"ed, so skip it here
536 * i.e. remove classless children
541 nd_device_unregister(dev
, ND_SYNC
);
545 void nvdimm_bus_unregister(struct nvdimm_bus
*nvdimm_bus
)
550 mutex_lock(&nvdimm_bus_list_mutex
);
551 list_del_init(&nvdimm_bus
->list
);
552 mutex_unlock(&nvdimm_bus_list_mutex
);
555 device_for_each_child(&nvdimm_bus
->dev
, NULL
, child_unregister
);
556 free_poison_list(&nvdimm_bus
->poison_list
);
557 nvdimm_bus_destroy_ndctl(nvdimm_bus
);
559 device_unregister(&nvdimm_bus
->dev
);
561 EXPORT_SYMBOL_GPL(nvdimm_bus_unregister
);
563 #ifdef CONFIG_BLK_DEV_INTEGRITY
564 int nd_integrity_init(struct gendisk
*disk
, unsigned long meta_size
)
566 struct blk_integrity bi
;
572 bi
.tuple_size
= meta_size
;
573 bi
.tag_size
= meta_size
;
575 blk_integrity_register(disk
, &bi
);
576 blk_queue_max_integrity_segments(disk
->queue
, 1);
580 EXPORT_SYMBOL(nd_integrity_init
);
582 #else /* CONFIG_BLK_DEV_INTEGRITY */
583 int nd_integrity_init(struct gendisk
*disk
, unsigned long meta_size
)
587 EXPORT_SYMBOL(nd_integrity_init
);
591 static __init
int libnvdimm_init(void)
595 rc
= nvdimm_bus_init();
601 rc
= nd_region_init();
612 static __exit
void libnvdimm_exit(void)
614 WARN_ON(!list_empty(&nvdimm_bus_list
));
620 MODULE_LICENSE("GPL v2");
621 MODULE_AUTHOR("Intel Corporation");
622 subsys_initcall(libnvdimm_init
);
623 module_exit(libnvdimm_exit
);