1 #include <linux/errno.h>
2 #include <linux/numa.h>
3 #include <linux/slab.h>
4 #include <linux/rculist.h>
5 #include <linux/threads.h>
6 #include <linux/preempt.h>
7 #include <linux/irqflags.h>
8 #include <linux/vmalloc.h>
10 #include <linux/module.h>
11 #include <linux/device-mapper.h>
16 #define DM_MSG_PREFIX "stats"
18 static int dm_stat_need_rcu_barrier
;
21 * Using 64-bit values to avoid overflow (which is a
22 * problem that block/genhd.c's IO accounting has).
24 struct dm_stat_percpu
{
25 unsigned long long sectors
[2];
26 unsigned long long ios
[2];
27 unsigned long long merges
[2];
28 unsigned long long ticks
[2];
29 unsigned long long io_ticks
[2];
30 unsigned long long io_ticks_total
;
31 unsigned long long time_in_queue
;
32 unsigned long long *histogram
;
35 struct dm_stat_shared
{
36 atomic_t in_flight
[2];
37 unsigned long long stamp
;
38 struct dm_stat_percpu tmp
;
42 struct list_head list_entry
;
49 unsigned n_histogram_entries
;
50 unsigned long long *histogram_boundaries
;
51 const char *program_id
;
53 struct rcu_head rcu_head
;
54 size_t shared_alloc_size
;
55 size_t percpu_alloc_size
;
56 size_t histogram_alloc_size
;
57 struct dm_stat_percpu
*stat_percpu
[NR_CPUS
];
58 struct dm_stat_shared stat_shared
[0];
61 #define STAT_PRECISE_TIMESTAMPS 1
63 struct dm_stats_last_position
{
69 * A typo on the command line could possibly make the kernel run out of memory
70 * and crash. To prevent the crash we account all used memory. We fail if we
71 * exhaust 1/4 of all memory or 1/2 of vmalloc space.
73 #define DM_STATS_MEMORY_FACTOR 4
74 #define DM_STATS_VMALLOC_FACTOR 2
76 static DEFINE_SPINLOCK(shared_memory_lock
);
78 static unsigned long shared_memory_amount
;
80 static bool __check_shared_memory(size_t alloc_size
)
84 a
= shared_memory_amount
+ alloc_size
;
85 if (a
< shared_memory_amount
)
87 if (a
>> PAGE_SHIFT
> totalram_pages
/ DM_STATS_MEMORY_FACTOR
)
90 if (a
> (VMALLOC_END
- VMALLOC_START
) / DM_STATS_VMALLOC_FACTOR
)
96 static bool check_shared_memory(size_t alloc_size
)
100 spin_lock_irq(&shared_memory_lock
);
102 ret
= __check_shared_memory(alloc_size
);
104 spin_unlock_irq(&shared_memory_lock
);
109 static bool claim_shared_memory(size_t alloc_size
)
111 spin_lock_irq(&shared_memory_lock
);
113 if (!__check_shared_memory(alloc_size
)) {
114 spin_unlock_irq(&shared_memory_lock
);
118 shared_memory_amount
+= alloc_size
;
120 spin_unlock_irq(&shared_memory_lock
);
125 static void free_shared_memory(size_t alloc_size
)
129 spin_lock_irqsave(&shared_memory_lock
, flags
);
131 if (WARN_ON_ONCE(shared_memory_amount
< alloc_size
)) {
132 spin_unlock_irqrestore(&shared_memory_lock
, flags
);
133 DMCRIT("Memory usage accounting bug.");
137 shared_memory_amount
-= alloc_size
;
139 spin_unlock_irqrestore(&shared_memory_lock
, flags
);
142 static void *dm_kvzalloc(size_t alloc_size
, int node
)
146 if (!claim_shared_memory(alloc_size
))
149 if (alloc_size
<= KMALLOC_MAX_SIZE
) {
150 p
= kzalloc_node(alloc_size
, GFP_KERNEL
| __GFP_NORETRY
| __GFP_NOMEMALLOC
| __GFP_NOWARN
, node
);
154 p
= vzalloc_node(alloc_size
, node
);
158 free_shared_memory(alloc_size
);
163 static void dm_kvfree(void *ptr
, size_t alloc_size
)
168 free_shared_memory(alloc_size
);
173 static void dm_stat_free(struct rcu_head
*head
)
176 struct dm_stat
*s
= container_of(head
, struct dm_stat
, rcu_head
);
178 kfree(s
->program_id
);
180 for_each_possible_cpu(cpu
) {
181 dm_kvfree(s
->stat_percpu
[cpu
][0].histogram
, s
->histogram_alloc_size
);
182 dm_kvfree(s
->stat_percpu
[cpu
], s
->percpu_alloc_size
);
184 dm_kvfree(s
->stat_shared
[0].tmp
.histogram
, s
->histogram_alloc_size
);
185 dm_kvfree(s
, s
->shared_alloc_size
);
188 static int dm_stat_in_flight(struct dm_stat_shared
*shared
)
190 return atomic_read(&shared
->in_flight
[READ
]) +
191 atomic_read(&shared
->in_flight
[WRITE
]);
194 void dm_stats_init(struct dm_stats
*stats
)
197 struct dm_stats_last_position
*last
;
199 mutex_init(&stats
->mutex
);
200 INIT_LIST_HEAD(&stats
->list
);
201 stats
->last
= alloc_percpu(struct dm_stats_last_position
);
202 for_each_possible_cpu(cpu
) {
203 last
= per_cpu_ptr(stats
->last
, cpu
);
204 last
->last_sector
= (sector_t
)ULLONG_MAX
;
205 last
->last_rw
= UINT_MAX
;
209 void dm_stats_cleanup(struct dm_stats
*stats
)
213 struct dm_stat_shared
*shared
;
215 while (!list_empty(&stats
->list
)) {
216 s
= container_of(stats
->list
.next
, struct dm_stat
, list_entry
);
217 list_del(&s
->list_entry
);
218 for (ni
= 0; ni
< s
->n_entries
; ni
++) {
219 shared
= &s
->stat_shared
[ni
];
220 if (WARN_ON(dm_stat_in_flight(shared
))) {
221 DMCRIT("leaked in-flight counter at index %lu "
222 "(start %llu, end %llu, step %llu): reads %d, writes %d",
224 (unsigned long long)s
->start
,
225 (unsigned long long)s
->end
,
226 (unsigned long long)s
->step
,
227 atomic_read(&shared
->in_flight
[READ
]),
228 atomic_read(&shared
->in_flight
[WRITE
]));
231 dm_stat_free(&s
->rcu_head
);
233 free_percpu(stats
->last
);
236 static int dm_stats_create(struct dm_stats
*stats
, sector_t start
, sector_t end
,
237 sector_t step
, unsigned stat_flags
,
238 unsigned n_histogram_entries
,
239 unsigned long long *histogram_boundaries
,
240 const char *program_id
, const char *aux_data
,
241 void (*suspend_callback
)(struct mapped_device
*),
242 void (*resume_callback
)(struct mapped_device
*),
243 struct mapped_device
*md
)
246 struct dm_stat
*s
, *tmp_s
;
249 size_t shared_alloc_size
;
250 size_t percpu_alloc_size
;
251 size_t histogram_alloc_size
;
252 struct dm_stat_percpu
*p
;
257 if (end
< start
|| !step
)
260 n_entries
= end
- start
;
261 if (dm_sector_div64(n_entries
, step
))
264 if (n_entries
!= (size_t)n_entries
|| !(size_t)(n_entries
+ 1))
267 shared_alloc_size
= sizeof(struct dm_stat
) + (size_t)n_entries
* sizeof(struct dm_stat_shared
);
268 if ((shared_alloc_size
- sizeof(struct dm_stat
)) / sizeof(struct dm_stat_shared
) != n_entries
)
271 percpu_alloc_size
= (size_t)n_entries
* sizeof(struct dm_stat_percpu
);
272 if (percpu_alloc_size
/ sizeof(struct dm_stat_percpu
) != n_entries
)
275 histogram_alloc_size
= (n_histogram_entries
+ 1) * (size_t)n_entries
* sizeof(unsigned long long);
276 if (histogram_alloc_size
/ (n_histogram_entries
+ 1) != (size_t)n_entries
* sizeof(unsigned long long))
279 if (!check_shared_memory(shared_alloc_size
+ histogram_alloc_size
+
280 num_possible_cpus() * (percpu_alloc_size
+ histogram_alloc_size
)))
283 s
= dm_kvzalloc(shared_alloc_size
, NUMA_NO_NODE
);
287 s
->stat_flags
= stat_flags
;
288 s
->n_entries
= n_entries
;
292 s
->shared_alloc_size
= shared_alloc_size
;
293 s
->percpu_alloc_size
= percpu_alloc_size
;
294 s
->histogram_alloc_size
= histogram_alloc_size
;
296 s
->n_histogram_entries
= n_histogram_entries
;
297 s
->histogram_boundaries
= kmemdup(histogram_boundaries
,
298 s
->n_histogram_entries
* sizeof(unsigned long long), GFP_KERNEL
);
299 if (!s
->histogram_boundaries
) {
304 s
->program_id
= kstrdup(program_id
, GFP_KERNEL
);
305 if (!s
->program_id
) {
309 s
->aux_data
= kstrdup(aux_data
, GFP_KERNEL
);
315 for (ni
= 0; ni
< n_entries
; ni
++) {
316 atomic_set(&s
->stat_shared
[ni
].in_flight
[READ
], 0);
317 atomic_set(&s
->stat_shared
[ni
].in_flight
[WRITE
], 0);
320 if (s
->n_histogram_entries
) {
321 unsigned long long *hi
;
322 hi
= dm_kvzalloc(s
->histogram_alloc_size
, NUMA_NO_NODE
);
327 for (ni
= 0; ni
< n_entries
; ni
++) {
328 s
->stat_shared
[ni
].tmp
.histogram
= hi
;
329 hi
+= s
->n_histogram_entries
+ 1;
333 for_each_possible_cpu(cpu
) {
334 p
= dm_kvzalloc(percpu_alloc_size
, cpu_to_node(cpu
));
339 s
->stat_percpu
[cpu
] = p
;
340 if (s
->n_histogram_entries
) {
341 unsigned long long *hi
;
342 hi
= dm_kvzalloc(s
->histogram_alloc_size
, cpu_to_node(cpu
));
347 for (ni
= 0; ni
< n_entries
; ni
++) {
348 p
[ni
].histogram
= hi
;
349 hi
+= s
->n_histogram_entries
+ 1;
355 * Suspend/resume to make sure there is no i/o in flight,
356 * so that newly created statistics will be exact.
358 * (note: we couldn't suspend earlier because we must not
359 * allocate memory while suspended)
361 suspend_callback(md
);
363 mutex_lock(&stats
->mutex
);
365 list_for_each(l
, &stats
->list
) {
366 tmp_s
= container_of(l
, struct dm_stat
, list_entry
);
367 if (WARN_ON(tmp_s
->id
< s
->id
)) {
369 goto out_unlock_resume
;
371 if (tmp_s
->id
> s
->id
)
373 if (unlikely(s
->id
== INT_MAX
)) {
375 goto out_unlock_resume
;
380 list_add_tail_rcu(&s
->list_entry
, l
);
381 mutex_unlock(&stats
->mutex
);
388 mutex_unlock(&stats
->mutex
);
391 dm_stat_free(&s
->rcu_head
);
395 static struct dm_stat
*__dm_stats_find(struct dm_stats
*stats
, int id
)
399 list_for_each_entry(s
, &stats
->list
, list_entry
) {
409 static int dm_stats_delete(struct dm_stats
*stats
, int id
)
414 mutex_lock(&stats
->mutex
);
416 s
= __dm_stats_find(stats
, id
);
418 mutex_unlock(&stats
->mutex
);
422 list_del_rcu(&s
->list_entry
);
423 mutex_unlock(&stats
->mutex
);
426 * vfree can't be called from RCU callback
428 for_each_possible_cpu(cpu
)
429 if (is_vmalloc_addr(s
->stat_percpu
) ||
430 is_vmalloc_addr(s
->stat_percpu
[cpu
][0].histogram
))
432 if (is_vmalloc_addr(s
) ||
433 is_vmalloc_addr(s
->stat_shared
[0].tmp
.histogram
)) {
435 synchronize_rcu_expedited();
436 dm_stat_free(&s
->rcu_head
);
438 ACCESS_ONCE(dm_stat_need_rcu_barrier
) = 1;
439 call_rcu(&s
->rcu_head
, dm_stat_free
);
444 static int dm_stats_list(struct dm_stats
*stats
, const char *program
,
445 char *result
, unsigned maxlen
)
453 * <region_id>: <start_sector>+<length> <step> <program_id> <aux_data>
456 mutex_lock(&stats
->mutex
);
457 list_for_each_entry(s
, &stats
->list
, list_entry
) {
458 if (!program
|| !strcmp(program
, s
->program_id
)) {
459 len
= s
->end
- s
->start
;
460 DMEMIT("%d: %llu+%llu %llu %s %s\n", s
->id
,
461 (unsigned long long)s
->start
,
462 (unsigned long long)len
,
463 (unsigned long long)s
->step
,
468 mutex_unlock(&stats
->mutex
);
473 static void dm_stat_round(struct dm_stat
*s
, struct dm_stat_shared
*shared
,
474 struct dm_stat_percpu
*p
)
477 * This is racy, but so is part_round_stats_single.
479 unsigned long long now
, difference
;
480 unsigned in_flight_read
, in_flight_write
;
482 if (likely(!(s
->stat_flags
& STAT_PRECISE_TIMESTAMPS
)))
485 now
= ktime_to_ns(ktime_get());
487 difference
= now
- shared
->stamp
;
491 in_flight_read
= (unsigned)atomic_read(&shared
->in_flight
[READ
]);
492 in_flight_write
= (unsigned)atomic_read(&shared
->in_flight
[WRITE
]);
494 p
->io_ticks
[READ
] += difference
;
496 p
->io_ticks
[WRITE
] += difference
;
497 if (in_flight_read
+ in_flight_write
) {
498 p
->io_ticks_total
+= difference
;
499 p
->time_in_queue
+= (in_flight_read
+ in_flight_write
) * difference
;
504 static void dm_stat_for_entry(struct dm_stat
*s
, size_t entry
,
505 unsigned long bi_rw
, sector_t len
,
506 struct dm_stats_aux
*stats_aux
, bool end
,
507 unsigned long duration_jiffies
)
509 unsigned long idx
= bi_rw
& REQ_WRITE
;
510 struct dm_stat_shared
*shared
= &s
->stat_shared
[entry
];
511 struct dm_stat_percpu
*p
;
514 * For strict correctness we should use local_irq_save/restore
515 * instead of preempt_disable/enable.
517 * preempt_disable/enable is racy if the driver finishes bios
518 * from non-interrupt context as well as from interrupt context
519 * or from more different interrupts.
521 * On 64-bit architectures the race only results in not counting some
522 * events, so it is acceptable. On 32-bit architectures the race could
523 * cause the counter going off by 2^32, so we need to do proper locking
526 * part_stat_lock()/part_stat_unlock() have this race too.
528 #if BITS_PER_LONG == 32
530 local_irq_save(flags
);
534 p
= &s
->stat_percpu
[smp_processor_id()][entry
];
537 dm_stat_round(s
, shared
, p
);
538 atomic_inc(&shared
->in_flight
[idx
]);
540 unsigned long long duration
;
541 dm_stat_round(s
, shared
, p
);
542 atomic_dec(&shared
->in_flight
[idx
]);
543 p
->sectors
[idx
] += len
;
545 p
->merges
[idx
] += stats_aux
->merged
;
546 if (!(s
->stat_flags
& STAT_PRECISE_TIMESTAMPS
)) {
547 p
->ticks
[idx
] += duration_jiffies
;
548 duration
= jiffies_to_msecs(duration_jiffies
);
550 p
->ticks
[idx
] += stats_aux
->duration_ns
;
551 duration
= stats_aux
->duration_ns
;
553 if (s
->n_histogram_entries
) {
554 unsigned lo
= 0, hi
= s
->n_histogram_entries
+ 1;
555 while (lo
+ 1 < hi
) {
556 unsigned mid
= (lo
+ hi
) / 2;
557 if (s
->histogram_boundaries
[mid
- 1] > duration
) {
568 #if BITS_PER_LONG == 32
569 local_irq_restore(flags
);
575 static void __dm_stat_bio(struct dm_stat
*s
, unsigned long bi_rw
,
576 sector_t bi_sector
, sector_t end_sector
,
577 bool end
, unsigned long duration_jiffies
,
578 struct dm_stats_aux
*stats_aux
)
580 sector_t rel_sector
, offset
, todo
, fragment_len
;
583 if (end_sector
<= s
->start
|| bi_sector
>= s
->end
)
585 if (unlikely(bi_sector
< s
->start
)) {
587 todo
= end_sector
- s
->start
;
589 rel_sector
= bi_sector
- s
->start
;
590 todo
= end_sector
- bi_sector
;
592 if (unlikely(end_sector
> s
->end
))
593 todo
-= (end_sector
- s
->end
);
595 offset
= dm_sector_div64(rel_sector
, s
->step
);
598 if (WARN_ON_ONCE(entry
>= s
->n_entries
)) {
599 DMCRIT("Invalid area access in region id %d", s
->id
);
603 if (fragment_len
> s
->step
- offset
)
604 fragment_len
= s
->step
- offset
;
605 dm_stat_for_entry(s
, entry
, bi_rw
, fragment_len
,
606 stats_aux
, end
, duration_jiffies
);
607 todo
-= fragment_len
;
610 } while (unlikely(todo
!= 0));
613 void dm_stats_account_io(struct dm_stats
*stats
, unsigned long bi_rw
,
614 sector_t bi_sector
, unsigned bi_sectors
, bool end
,
615 unsigned long duration_jiffies
,
616 struct dm_stats_aux
*stats_aux
)
620 struct dm_stats_last_position
*last
;
621 bool got_precise_time
;
623 if (unlikely(!bi_sectors
))
626 end_sector
= bi_sector
+ bi_sectors
;
630 * A race condition can at worst result in the merged flag being
631 * misrepresented, so we don't have to disable preemption here.
633 last
= raw_cpu_ptr(stats
->last
);
635 (bi_sector
== (ACCESS_ONCE(last
->last_sector
) &&
636 ((bi_rw
& (REQ_WRITE
| REQ_DISCARD
)) ==
637 (ACCESS_ONCE(last
->last_rw
) & (REQ_WRITE
| REQ_DISCARD
)))
639 ACCESS_ONCE(last
->last_sector
) = end_sector
;
640 ACCESS_ONCE(last
->last_rw
) = bi_rw
;
645 got_precise_time
= false;
646 list_for_each_entry_rcu(s
, &stats
->list
, list_entry
) {
647 if (s
->stat_flags
& STAT_PRECISE_TIMESTAMPS
&& !got_precise_time
) {
649 stats_aux
->duration_ns
= ktime_to_ns(ktime_get());
651 stats_aux
->duration_ns
= ktime_to_ns(ktime_get()) - stats_aux
->duration_ns
;
652 got_precise_time
= true;
654 __dm_stat_bio(s
, bi_rw
, bi_sector
, end_sector
, end
, duration_jiffies
, stats_aux
);
660 static void __dm_stat_init_temporary_percpu_totals(struct dm_stat_shared
*shared
,
661 struct dm_stat
*s
, size_t x
)
664 struct dm_stat_percpu
*p
;
667 p
= &s
->stat_percpu
[smp_processor_id()][x
];
668 dm_stat_round(s
, shared
, p
);
671 shared
->tmp
.sectors
[READ
] = 0;
672 shared
->tmp
.sectors
[WRITE
] = 0;
673 shared
->tmp
.ios
[READ
] = 0;
674 shared
->tmp
.ios
[WRITE
] = 0;
675 shared
->tmp
.merges
[READ
] = 0;
676 shared
->tmp
.merges
[WRITE
] = 0;
677 shared
->tmp
.ticks
[READ
] = 0;
678 shared
->tmp
.ticks
[WRITE
] = 0;
679 shared
->tmp
.io_ticks
[READ
] = 0;
680 shared
->tmp
.io_ticks
[WRITE
] = 0;
681 shared
->tmp
.io_ticks_total
= 0;
682 shared
->tmp
.time_in_queue
= 0;
684 if (s
->n_histogram_entries
)
685 memset(shared
->tmp
.histogram
, 0, (s
->n_histogram_entries
+ 1) * sizeof(unsigned long long));
687 for_each_possible_cpu(cpu
) {
688 p
= &s
->stat_percpu
[cpu
][x
];
689 shared
->tmp
.sectors
[READ
] += ACCESS_ONCE(p
->sectors
[READ
]);
690 shared
->tmp
.sectors
[WRITE
] += ACCESS_ONCE(p
->sectors
[WRITE
]);
691 shared
->tmp
.ios
[READ
] += ACCESS_ONCE(p
->ios
[READ
]);
692 shared
->tmp
.ios
[WRITE
] += ACCESS_ONCE(p
->ios
[WRITE
]);
693 shared
->tmp
.merges
[READ
] += ACCESS_ONCE(p
->merges
[READ
]);
694 shared
->tmp
.merges
[WRITE
] += ACCESS_ONCE(p
->merges
[WRITE
]);
695 shared
->tmp
.ticks
[READ
] += ACCESS_ONCE(p
->ticks
[READ
]);
696 shared
->tmp
.ticks
[WRITE
] += ACCESS_ONCE(p
->ticks
[WRITE
]);
697 shared
->tmp
.io_ticks
[READ
] += ACCESS_ONCE(p
->io_ticks
[READ
]);
698 shared
->tmp
.io_ticks
[WRITE
] += ACCESS_ONCE(p
->io_ticks
[WRITE
]);
699 shared
->tmp
.io_ticks_total
+= ACCESS_ONCE(p
->io_ticks_total
);
700 shared
->tmp
.time_in_queue
+= ACCESS_ONCE(p
->time_in_queue
);
701 if (s
->n_histogram_entries
) {
703 for (i
= 0; i
< s
->n_histogram_entries
+ 1; i
++)
704 shared
->tmp
.histogram
[i
] += ACCESS_ONCE(p
->histogram
[i
]);
709 static void __dm_stat_clear(struct dm_stat
*s
, size_t idx_start
, size_t idx_end
,
710 bool init_tmp_percpu_totals
)
713 struct dm_stat_shared
*shared
;
714 struct dm_stat_percpu
*p
;
716 for (x
= idx_start
; x
< idx_end
; x
++) {
717 shared
= &s
->stat_shared
[x
];
718 if (init_tmp_percpu_totals
)
719 __dm_stat_init_temporary_percpu_totals(shared
, s
, x
);
721 p
= &s
->stat_percpu
[smp_processor_id()][x
];
722 p
->sectors
[READ
] -= shared
->tmp
.sectors
[READ
];
723 p
->sectors
[WRITE
] -= shared
->tmp
.sectors
[WRITE
];
724 p
->ios
[READ
] -= shared
->tmp
.ios
[READ
];
725 p
->ios
[WRITE
] -= shared
->tmp
.ios
[WRITE
];
726 p
->merges
[READ
] -= shared
->tmp
.merges
[READ
];
727 p
->merges
[WRITE
] -= shared
->tmp
.merges
[WRITE
];
728 p
->ticks
[READ
] -= shared
->tmp
.ticks
[READ
];
729 p
->ticks
[WRITE
] -= shared
->tmp
.ticks
[WRITE
];
730 p
->io_ticks
[READ
] -= shared
->tmp
.io_ticks
[READ
];
731 p
->io_ticks
[WRITE
] -= shared
->tmp
.io_ticks
[WRITE
];
732 p
->io_ticks_total
-= shared
->tmp
.io_ticks_total
;
733 p
->time_in_queue
-= shared
->tmp
.time_in_queue
;
735 if (s
->n_histogram_entries
) {
737 for (i
= 0; i
< s
->n_histogram_entries
+ 1; i
++) {
739 p
= &s
->stat_percpu
[smp_processor_id()][x
];
740 p
->histogram
[i
] -= shared
->tmp
.histogram
[i
];
747 static int dm_stats_clear(struct dm_stats
*stats
, int id
)
751 mutex_lock(&stats
->mutex
);
753 s
= __dm_stats_find(stats
, id
);
755 mutex_unlock(&stats
->mutex
);
759 __dm_stat_clear(s
, 0, s
->n_entries
, true);
761 mutex_unlock(&stats
->mutex
);
767 * This is like jiffies_to_msec, but works for 64-bit values.
769 static unsigned long long dm_jiffies_to_msec64(struct dm_stat
*s
, unsigned long long j
)
771 unsigned long long result
;
774 if (s
->stat_flags
& STAT_PRECISE_TIMESTAMPS
)
779 result
= jiffies_to_msecs(j
& 0x3fffff);
781 mult
= jiffies_to_msecs(1 << 22);
782 result
+= (unsigned long long)mult
* (unsigned long long)jiffies_to_msecs((j
>> 22) & 0x3fffff);
785 result
+= (unsigned long long)mult
* (unsigned long long)mult
* (unsigned long long)jiffies_to_msecs(j
>> 44);
790 static int dm_stats_print(struct dm_stats
*stats
, int id
,
791 size_t idx_start
, size_t idx_len
,
792 bool clear
, char *result
, unsigned maxlen
)
797 sector_t start
, end
, step
;
799 struct dm_stat_shared
*shared
;
803 * <start_sector>+<length> counters
806 mutex_lock(&stats
->mutex
);
808 s
= __dm_stats_find(stats
, id
);
810 mutex_unlock(&stats
->mutex
);
814 idx_end
= idx_start
+ idx_len
;
815 if (idx_end
< idx_start
||
816 idx_end
> s
->n_entries
)
817 idx_end
= s
->n_entries
;
819 if (idx_start
> idx_end
)
823 start
= s
->start
+ (step
* idx_start
);
825 for (x
= idx_start
; x
< idx_end
; x
++, start
= end
) {
826 shared
= &s
->stat_shared
[x
];
828 if (unlikely(end
> s
->end
))
831 __dm_stat_init_temporary_percpu_totals(shared
, s
, x
);
833 DMEMIT("%llu+%llu %llu %llu %llu %llu %llu %llu %llu %llu %d %llu %llu %llu %llu",
834 (unsigned long long)start
,
835 (unsigned long long)step
,
836 shared
->tmp
.ios
[READ
],
837 shared
->tmp
.merges
[READ
],
838 shared
->tmp
.sectors
[READ
],
839 dm_jiffies_to_msec64(s
, shared
->tmp
.ticks
[READ
]),
840 shared
->tmp
.ios
[WRITE
],
841 shared
->tmp
.merges
[WRITE
],
842 shared
->tmp
.sectors
[WRITE
],
843 dm_jiffies_to_msec64(s
, shared
->tmp
.ticks
[WRITE
]),
844 dm_stat_in_flight(shared
),
845 dm_jiffies_to_msec64(s
, shared
->tmp
.io_ticks_total
),
846 dm_jiffies_to_msec64(s
, shared
->tmp
.time_in_queue
),
847 dm_jiffies_to_msec64(s
, shared
->tmp
.io_ticks
[READ
]),
848 dm_jiffies_to_msec64(s
, shared
->tmp
.io_ticks
[WRITE
]));
849 if (s
->n_histogram_entries
) {
851 for (i
= 0; i
< s
->n_histogram_entries
+ 1; i
++) {
852 DMEMIT("%s%llu", !i
? " " : ":", shared
->tmp
.histogram
[i
]);
857 if (unlikely(sz
+ 1 >= maxlen
))
858 goto buffer_overflow
;
862 __dm_stat_clear(s
, idx_start
, idx_end
, false);
865 mutex_unlock(&stats
->mutex
);
870 static int dm_stats_set_aux(struct dm_stats
*stats
, int id
, const char *aux_data
)
873 const char *new_aux_data
;
875 mutex_lock(&stats
->mutex
);
877 s
= __dm_stats_find(stats
, id
);
879 mutex_unlock(&stats
->mutex
);
883 new_aux_data
= kstrdup(aux_data
, GFP_KERNEL
);
885 mutex_unlock(&stats
->mutex
);
890 s
->aux_data
= new_aux_data
;
892 mutex_unlock(&stats
->mutex
);
897 static int parse_histogram(const char *h
, unsigned *n_histogram_entries
,
898 unsigned long long **histogram_boundaries
)
902 unsigned long long last
;
904 *n_histogram_entries
= 1;
907 (*n_histogram_entries
)++;
909 *histogram_boundaries
= kmalloc(*n_histogram_entries
* sizeof(unsigned long long), GFP_KERNEL
);
910 if (!*histogram_boundaries
)
916 unsigned long long hi
;
919 s
= sscanf(h
, "%llu%c", &hi
, &ch
);
920 if (!s
|| (s
== 2 && ch
!= ','))
925 (*histogram_boundaries
)[n
] = hi
;
928 h
= strchr(h
, ',') + 1;
933 static int message_stats_create(struct mapped_device
*md
,
934 unsigned argc
, char **argv
,
935 char *result
, unsigned maxlen
)
940 unsigned long long start
, end
, len
, step
;
942 const char *program_id
, *aux_data
;
943 unsigned stat_flags
= 0;
945 unsigned n_histogram_entries
= 0;
946 unsigned long long *histogram_boundaries
= NULL
;
948 struct dm_arg_set as
, as_backup
;
950 unsigned feature_args
;
954 * <range> <step> [<extra_parameters> <parameters>] [<program_id> [<aux_data>]]
962 dm_consume_args(&as
, 1);
964 a
= dm_shift_arg(&as
);
965 if (!strcmp(a
, "-")) {
967 len
= dm_get_size(md
);
970 } else if (sscanf(a
, "%llu+%llu%c", &start
, &len
, &dummy
) != 2 ||
971 start
!= (sector_t
)start
|| len
!= (sector_t
)len
)
978 a
= dm_shift_arg(&as
);
979 if (sscanf(a
, "/%u%c", &divisor
, &dummy
) == 1) {
983 if (do_div(step
, divisor
))
987 } else if (sscanf(a
, "%llu%c", &step
, &dummy
) != 1 ||
988 step
!= (sector_t
)step
|| !step
)
992 a
= dm_shift_arg(&as
);
993 if (a
&& sscanf(a
, "%u%c", &feature_args
, &dummy
) == 1) {
994 while (feature_args
--) {
995 a
= dm_shift_arg(&as
);
998 if (!strcasecmp(a
, "precise_timestamps"))
999 stat_flags
|= STAT_PRECISE_TIMESTAMPS
;
1000 else if (!strncasecmp(a
, "histogram:", 10)) {
1001 if (n_histogram_entries
)
1003 if ((r
= parse_histogram(a
+ 10, &n_histogram_entries
, &histogram_boundaries
)))
1015 a
= dm_shift_arg(&as
);
1019 a
= dm_shift_arg(&as
);
1027 * If a buffer overflow happens after we created the region,
1028 * it's too late (the userspace would retry with a larger
1029 * buffer, but the region id that caused the overflow is already
1030 * leaked). So we must detect buffer overflow in advance.
1032 snprintf(result
, maxlen
, "%d", INT_MAX
);
1033 if (dm_message_test_buffer_overflow(result
, maxlen
)) {
1038 id
= dm_stats_create(dm_get_stats(md
), start
, end
, step
, stat_flags
,
1039 n_histogram_entries
, histogram_boundaries
, program_id
, aux_data
,
1040 dm_internal_suspend_fast
, dm_internal_resume_fast
, md
);
1046 snprintf(result
, maxlen
, "%d", id
);
1054 kfree(histogram_boundaries
);
1058 static int message_stats_delete(struct mapped_device
*md
,
1059 unsigned argc
, char **argv
)
1067 if (sscanf(argv
[1], "%d%c", &id
, &dummy
) != 1 || id
< 0)
1070 return dm_stats_delete(dm_get_stats(md
), id
);
1073 static int message_stats_clear(struct mapped_device
*md
,
1074 unsigned argc
, char **argv
)
1082 if (sscanf(argv
[1], "%d%c", &id
, &dummy
) != 1 || id
< 0)
1085 return dm_stats_clear(dm_get_stats(md
), id
);
1088 static int message_stats_list(struct mapped_device
*md
,
1089 unsigned argc
, char **argv
,
1090 char *result
, unsigned maxlen
)
1093 const char *program
= NULL
;
1095 if (argc
< 1 || argc
> 2)
1099 program
= kstrdup(argv
[1], GFP_KERNEL
);
1104 r
= dm_stats_list(dm_get_stats(md
), program
, result
, maxlen
);
1111 static int message_stats_print(struct mapped_device
*md
,
1112 unsigned argc
, char **argv
, bool clear
,
1113 char *result
, unsigned maxlen
)
1117 unsigned long idx_start
= 0, idx_len
= ULONG_MAX
;
1119 if (argc
!= 2 && argc
!= 4)
1122 if (sscanf(argv
[1], "%d%c", &id
, &dummy
) != 1 || id
< 0)
1126 if (strcmp(argv
[2], "-") &&
1127 sscanf(argv
[2], "%lu%c", &idx_start
, &dummy
) != 1)
1129 if (strcmp(argv
[3], "-") &&
1130 sscanf(argv
[3], "%lu%c", &idx_len
, &dummy
) != 1)
1134 return dm_stats_print(dm_get_stats(md
), id
, idx_start
, idx_len
, clear
,
1138 static int message_stats_set_aux(struct mapped_device
*md
,
1139 unsigned argc
, char **argv
)
1147 if (sscanf(argv
[1], "%d%c", &id
, &dummy
) != 1 || id
< 0)
1150 return dm_stats_set_aux(dm_get_stats(md
), id
, argv
[2]);
1153 int dm_stats_message(struct mapped_device
*md
, unsigned argc
, char **argv
,
1154 char *result
, unsigned maxlen
)
1158 /* All messages here must start with '@' */
1159 if (!strcasecmp(argv
[0], "@stats_create"))
1160 r
= message_stats_create(md
, argc
, argv
, result
, maxlen
);
1161 else if (!strcasecmp(argv
[0], "@stats_delete"))
1162 r
= message_stats_delete(md
, argc
, argv
);
1163 else if (!strcasecmp(argv
[0], "@stats_clear"))
1164 r
= message_stats_clear(md
, argc
, argv
);
1165 else if (!strcasecmp(argv
[0], "@stats_list"))
1166 r
= message_stats_list(md
, argc
, argv
, result
, maxlen
);
1167 else if (!strcasecmp(argv
[0], "@stats_print"))
1168 r
= message_stats_print(md
, argc
, argv
, false, result
, maxlen
);
1169 else if (!strcasecmp(argv
[0], "@stats_print_clear"))
1170 r
= message_stats_print(md
, argc
, argv
, true, result
, maxlen
);
1171 else if (!strcasecmp(argv
[0], "@stats_set_aux"))
1172 r
= message_stats_set_aux(md
, argc
, argv
);
1174 return 2; /* this wasn't a stats message */
1177 DMWARN("Invalid parameters for message %s", argv
[0]);
1182 int __init
dm_statistics_init(void)
1184 shared_memory_amount
= 0;
1185 dm_stat_need_rcu_barrier
= 0;
1189 void dm_statistics_exit(void)
1191 if (dm_stat_need_rcu_barrier
)
1193 if (WARN_ON(shared_memory_amount
))
1194 DMCRIT("shared_memory_amount leaked: %lu", shared_memory_amount
);
1197 module_param_named(stats_current_allocated_bytes
, shared_memory_amount
, ulong
, S_IRUGO
);
1198 MODULE_PARM_DESC(stats_current_allocated_bytes
, "Memory currently used by statistics");