1 #include <linux/kernel.h>
2 #include <linux/module.h>
3 #include <linux/backing-dev.h>
5 #include <linux/blkdev.h>
7 #include <linux/init.h>
8 #include <linux/slab.h>
9 #include <linux/workqueue.h>
10 #include <linux/smp.h>
12 #include <linux/blk-mq.h>
14 #include "blk-mq-tag.h"
16 static void blk_mq_sysfs_release(struct kobject
*kobj
)
20 struct blk_mq_ctx_sysfs_entry
{
21 struct attribute attr
;
22 ssize_t (*show
)(struct blk_mq_ctx
*, char *);
23 ssize_t (*store
)(struct blk_mq_ctx
*, const char *, size_t);
26 struct blk_mq_hw_ctx_sysfs_entry
{
27 struct attribute attr
;
28 ssize_t (*show
)(struct blk_mq_hw_ctx
*, char *);
29 ssize_t (*store
)(struct blk_mq_hw_ctx
*, const char *, size_t);
32 static ssize_t
blk_mq_sysfs_show(struct kobject
*kobj
, struct attribute
*attr
,
35 struct blk_mq_ctx_sysfs_entry
*entry
;
36 struct blk_mq_ctx
*ctx
;
37 struct request_queue
*q
;
40 entry
= container_of(attr
, struct blk_mq_ctx_sysfs_entry
, attr
);
41 ctx
= container_of(kobj
, struct blk_mq_ctx
, kobj
);
48 mutex_lock(&q
->sysfs_lock
);
49 if (!blk_queue_dying(q
))
50 res
= entry
->show(ctx
, page
);
51 mutex_unlock(&q
->sysfs_lock
);
55 static ssize_t
blk_mq_sysfs_store(struct kobject
*kobj
, struct attribute
*attr
,
56 const char *page
, size_t length
)
58 struct blk_mq_ctx_sysfs_entry
*entry
;
59 struct blk_mq_ctx
*ctx
;
60 struct request_queue
*q
;
63 entry
= container_of(attr
, struct blk_mq_ctx_sysfs_entry
, attr
);
64 ctx
= container_of(kobj
, struct blk_mq_ctx
, kobj
);
71 mutex_lock(&q
->sysfs_lock
);
72 if (!blk_queue_dying(q
))
73 res
= entry
->store(ctx
, page
, length
);
74 mutex_unlock(&q
->sysfs_lock
);
78 static ssize_t
blk_mq_hw_sysfs_show(struct kobject
*kobj
,
79 struct attribute
*attr
, char *page
)
81 struct blk_mq_hw_ctx_sysfs_entry
*entry
;
82 struct blk_mq_hw_ctx
*hctx
;
83 struct request_queue
*q
;
86 entry
= container_of(attr
, struct blk_mq_hw_ctx_sysfs_entry
, attr
);
87 hctx
= container_of(kobj
, struct blk_mq_hw_ctx
, kobj
);
94 mutex_lock(&q
->sysfs_lock
);
95 if (!blk_queue_dying(q
))
96 res
= entry
->show(hctx
, page
);
97 mutex_unlock(&q
->sysfs_lock
);
101 static ssize_t
blk_mq_hw_sysfs_store(struct kobject
*kobj
,
102 struct attribute
*attr
, const char *page
,
105 struct blk_mq_hw_ctx_sysfs_entry
*entry
;
106 struct blk_mq_hw_ctx
*hctx
;
107 struct request_queue
*q
;
110 entry
= container_of(attr
, struct blk_mq_hw_ctx_sysfs_entry
, attr
);
111 hctx
= container_of(kobj
, struct blk_mq_hw_ctx
, kobj
);
118 mutex_lock(&q
->sysfs_lock
);
119 if (!blk_queue_dying(q
))
120 res
= entry
->store(hctx
, page
, length
);
121 mutex_unlock(&q
->sysfs_lock
);
125 static ssize_t
blk_mq_sysfs_dispatched_show(struct blk_mq_ctx
*ctx
, char *page
)
127 return sprintf(page
, "%lu %lu\n", ctx
->rq_dispatched
[1],
128 ctx
->rq_dispatched
[0]);
131 static ssize_t
blk_mq_sysfs_merged_show(struct blk_mq_ctx
*ctx
, char *page
)
133 return sprintf(page
, "%lu\n", ctx
->rq_merged
);
136 static ssize_t
blk_mq_sysfs_completed_show(struct blk_mq_ctx
*ctx
, char *page
)
138 return sprintf(page
, "%lu %lu\n", ctx
->rq_completed
[1],
139 ctx
->rq_completed
[0]);
142 static ssize_t
sysfs_list_show(char *page
, struct list_head
*list
, char *msg
)
145 int len
= snprintf(page
, PAGE_SIZE
- 1, "%s:\n", msg
);
147 list_for_each_entry(rq
, list
, queuelist
) {
148 const int rq_len
= 2 * sizeof(rq
) + 2;
150 /* if the output will be truncated */
151 if (PAGE_SIZE
- 1 < len
+ rq_len
) {
152 /* backspacing if it can't hold '\t...\n' */
153 if (PAGE_SIZE
- 1 < len
+ 5)
155 len
+= snprintf(page
+ len
, PAGE_SIZE
- 1 - len
,
159 len
+= snprintf(page
+ len
, PAGE_SIZE
- 1 - len
,
166 static ssize_t
blk_mq_sysfs_rq_list_show(struct blk_mq_ctx
*ctx
, char *page
)
170 spin_lock(&ctx
->lock
);
171 ret
= sysfs_list_show(page
, &ctx
->rq_list
, "CTX pending");
172 spin_unlock(&ctx
->lock
);
177 static ssize_t
blk_mq_hw_sysfs_queued_show(struct blk_mq_hw_ctx
*hctx
,
180 return sprintf(page
, "%lu\n", hctx
->queued
);
183 static ssize_t
blk_mq_hw_sysfs_run_show(struct blk_mq_hw_ctx
*hctx
, char *page
)
185 return sprintf(page
, "%lu\n", hctx
->run
);
188 static ssize_t
blk_mq_hw_sysfs_dispatched_show(struct blk_mq_hw_ctx
*hctx
,
191 char *start_page
= page
;
194 page
+= sprintf(page
, "%8u\t%lu\n", 0U, hctx
->dispatched
[0]);
196 for (i
= 1; i
< BLK_MQ_MAX_DISPATCH_ORDER
; i
++) {
197 unsigned long d
= 1U << (i
- 1);
199 page
+= sprintf(page
, "%8lu\t%lu\n", d
, hctx
->dispatched
[i
]);
202 return page
- start_page
;
205 static ssize_t
blk_mq_hw_sysfs_rq_list_show(struct blk_mq_hw_ctx
*hctx
,
210 spin_lock(&hctx
->lock
);
211 ret
= sysfs_list_show(page
, &hctx
->dispatch
, "HCTX pending");
212 spin_unlock(&hctx
->lock
);
217 static ssize_t
blk_mq_hw_sysfs_tags_show(struct blk_mq_hw_ctx
*hctx
, char *page
)
219 return blk_mq_tag_sysfs_show(hctx
->tags
, page
);
222 static ssize_t
blk_mq_hw_sysfs_active_show(struct blk_mq_hw_ctx
*hctx
, char *page
)
224 return sprintf(page
, "%u\n", atomic_read(&hctx
->nr_active
));
227 static ssize_t
blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx
*hctx
, char *page
)
229 unsigned int i
, first
= 1;
232 for_each_cpu(i
, hctx
->cpumask
) {
234 ret
+= sprintf(ret
+ page
, "%u", i
);
236 ret
+= sprintf(ret
+ page
, ", %u", i
);
241 ret
+= sprintf(ret
+ page
, "\n");
245 static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_dispatched
= {
246 .attr
= {.name
= "dispatched", .mode
= S_IRUGO
},
247 .show
= blk_mq_sysfs_dispatched_show
,
249 static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_merged
= {
250 .attr
= {.name
= "merged", .mode
= S_IRUGO
},
251 .show
= blk_mq_sysfs_merged_show
,
253 static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_completed
= {
254 .attr
= {.name
= "completed", .mode
= S_IRUGO
},
255 .show
= blk_mq_sysfs_completed_show
,
257 static struct blk_mq_ctx_sysfs_entry blk_mq_sysfs_rq_list
= {
258 .attr
= {.name
= "rq_list", .mode
= S_IRUGO
},
259 .show
= blk_mq_sysfs_rq_list_show
,
262 static struct attribute
*default_ctx_attrs
[] = {
263 &blk_mq_sysfs_dispatched
.attr
,
264 &blk_mq_sysfs_merged
.attr
,
265 &blk_mq_sysfs_completed
.attr
,
266 &blk_mq_sysfs_rq_list
.attr
,
270 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_queued
= {
271 .attr
= {.name
= "queued", .mode
= S_IRUGO
},
272 .show
= blk_mq_hw_sysfs_queued_show
,
274 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_run
= {
275 .attr
= {.name
= "run", .mode
= S_IRUGO
},
276 .show
= blk_mq_hw_sysfs_run_show
,
278 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_dispatched
= {
279 .attr
= {.name
= "dispatched", .mode
= S_IRUGO
},
280 .show
= blk_mq_hw_sysfs_dispatched_show
,
282 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_active
= {
283 .attr
= {.name
= "active", .mode
= S_IRUGO
},
284 .show
= blk_mq_hw_sysfs_active_show
,
286 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_pending
= {
287 .attr
= {.name
= "pending", .mode
= S_IRUGO
},
288 .show
= blk_mq_hw_sysfs_rq_list_show
,
290 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_tags
= {
291 .attr
= {.name
= "tags", .mode
= S_IRUGO
},
292 .show
= blk_mq_hw_sysfs_tags_show
,
294 static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus
= {
295 .attr
= {.name
= "cpu_list", .mode
= S_IRUGO
},
296 .show
= blk_mq_hw_sysfs_cpus_show
,
299 static struct attribute
*default_hw_ctx_attrs
[] = {
300 &blk_mq_hw_sysfs_queued
.attr
,
301 &blk_mq_hw_sysfs_run
.attr
,
302 &blk_mq_hw_sysfs_dispatched
.attr
,
303 &blk_mq_hw_sysfs_pending
.attr
,
304 &blk_mq_hw_sysfs_tags
.attr
,
305 &blk_mq_hw_sysfs_cpus
.attr
,
306 &blk_mq_hw_sysfs_active
.attr
,
310 static const struct sysfs_ops blk_mq_sysfs_ops
= {
311 .show
= blk_mq_sysfs_show
,
312 .store
= blk_mq_sysfs_store
,
315 static const struct sysfs_ops blk_mq_hw_sysfs_ops
= {
316 .show
= blk_mq_hw_sysfs_show
,
317 .store
= blk_mq_hw_sysfs_store
,
320 static struct kobj_type blk_mq_ktype
= {
321 .sysfs_ops
= &blk_mq_sysfs_ops
,
322 .release
= blk_mq_sysfs_release
,
325 static struct kobj_type blk_mq_ctx_ktype
= {
326 .sysfs_ops
= &blk_mq_sysfs_ops
,
327 .default_attrs
= default_ctx_attrs
,
328 .release
= blk_mq_sysfs_release
,
331 static struct kobj_type blk_mq_hw_ktype
= {
332 .sysfs_ops
= &blk_mq_hw_sysfs_ops
,
333 .default_attrs
= default_hw_ctx_attrs
,
334 .release
= blk_mq_sysfs_release
,
337 static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx
*hctx
)
339 struct blk_mq_ctx
*ctx
;
345 hctx_for_each_ctx(hctx
, ctx
, i
)
346 kobject_del(&ctx
->kobj
);
348 kobject_del(&hctx
->kobj
);
351 static int blk_mq_register_hctx(struct blk_mq_hw_ctx
*hctx
)
353 struct request_queue
*q
= hctx
->queue
;
354 struct blk_mq_ctx
*ctx
;
360 ret
= kobject_add(&hctx
->kobj
, &q
->mq_kobj
, "%u", hctx
->queue_num
);
364 hctx_for_each_ctx(hctx
, ctx
, i
) {
365 ret
= kobject_add(&ctx
->kobj
, &hctx
->kobj
, "cpu%u", ctx
->cpu
);
373 void blk_mq_unregister_disk(struct gendisk
*disk
)
375 struct request_queue
*q
= disk
->queue
;
376 struct blk_mq_hw_ctx
*hctx
;
377 struct blk_mq_ctx
*ctx
;
380 blk_mq_disable_hotplug();
382 queue_for_each_hw_ctx(q
, hctx
, i
) {
383 blk_mq_unregister_hctx(hctx
);
385 hctx_for_each_ctx(hctx
, ctx
, j
)
386 kobject_put(&ctx
->kobj
);
388 kobject_put(&hctx
->kobj
);
391 kobject_uevent(&q
->mq_kobj
, KOBJ_REMOVE
);
392 kobject_del(&q
->mq_kobj
);
393 kobject_put(&q
->mq_kobj
);
395 kobject_put(&disk_to_dev(disk
)->kobj
);
397 q
->mq_sysfs_init_done
= false;
398 blk_mq_enable_hotplug();
401 static void blk_mq_sysfs_init(struct request_queue
*q
)
403 struct blk_mq_hw_ctx
*hctx
;
404 struct blk_mq_ctx
*ctx
;
407 kobject_init(&q
->mq_kobj
, &blk_mq_ktype
);
409 queue_for_each_hw_ctx(q
, hctx
, i
)
410 kobject_init(&hctx
->kobj
, &blk_mq_hw_ktype
);
412 queue_for_each_ctx(q
, ctx
, i
)
413 kobject_init(&ctx
->kobj
, &blk_mq_ctx_ktype
);
416 /* see blk_register_queue() */
417 void blk_mq_finish_init(struct request_queue
*q
)
419 percpu_ref_switch_to_percpu(&q
->mq_usage_counter
);
422 int blk_mq_register_disk(struct gendisk
*disk
)
424 struct device
*dev
= disk_to_dev(disk
);
425 struct request_queue
*q
= disk
->queue
;
426 struct blk_mq_hw_ctx
*hctx
;
429 blk_mq_disable_hotplug();
431 blk_mq_sysfs_init(q
);
433 ret
= kobject_add(&q
->mq_kobj
, kobject_get(&dev
->kobj
), "%s", "mq");
437 kobject_uevent(&q
->mq_kobj
, KOBJ_ADD
);
439 queue_for_each_hw_ctx(q
, hctx
, i
) {
440 ret
= blk_mq_register_hctx(hctx
);
446 blk_mq_unregister_disk(disk
);
448 q
->mq_sysfs_init_done
= true;
450 blk_mq_enable_hotplug();
454 EXPORT_SYMBOL_GPL(blk_mq_register_disk
);
456 void blk_mq_sysfs_unregister(struct request_queue
*q
)
458 struct blk_mq_hw_ctx
*hctx
;
461 if (!q
->mq_sysfs_init_done
)
464 queue_for_each_hw_ctx(q
, hctx
, i
)
465 blk_mq_unregister_hctx(hctx
);
468 int blk_mq_sysfs_register(struct request_queue
*q
)
470 struct blk_mq_hw_ctx
*hctx
;
473 if (!q
->mq_sysfs_init_done
)
476 queue_for_each_hw_ctx(q
, hctx
, i
) {
477 ret
= blk_mq_register_hctx(hctx
);