1 #include <linux/module.h>
3 #include <linux/moduleparam.h>
4 #include <linux/sched.h>
6 #include <linux/blkdev.h>
7 #include <linux/init.h>
8 #include <linux/slab.h>
9 #include <linux/blk-mq.h>
10 #include <linux/hrtimer.h>
11 #include <linux/lightnvm.h>
14 struct list_head list
;
15 struct llist_node ll_list
;
16 struct call_single_data csd
;
20 struct nullb_queue
*nq
;
25 unsigned long *tag_map
;
26 wait_queue_head_t wait
;
27 unsigned int queue_depth
;
29 struct nullb_cmd
*cmds
;
33 struct list_head list
;
35 struct request_queue
*q
;
37 struct blk_mq_tag_set tag_set
;
39 unsigned int queue_depth
;
42 struct nullb_queue
*queues
;
43 unsigned int nr_queues
;
44 char disk_name
[DISK_NAME_LEN
];
47 static LIST_HEAD(nullb_list
);
48 static struct mutex lock
;
49 static int null_major
;
50 static int nullb_indexes
;
51 static struct kmem_cache
*ppa_cache
;
65 static int submit_queues
;
66 module_param(submit_queues
, int, S_IRUGO
);
67 MODULE_PARM_DESC(submit_queues
, "Number of submission queues");
69 static int home_node
= NUMA_NO_NODE
;
70 module_param(home_node
, int, S_IRUGO
);
71 MODULE_PARM_DESC(home_node
, "Home node for the device");
73 static int queue_mode
= NULL_Q_MQ
;
75 static int null_param_store_val(const char *str
, int *val
, int min
, int max
)
79 ret
= kstrtoint(str
, 10, &new_val
);
83 if (new_val
< min
|| new_val
> max
)
90 static int null_set_queue_mode(const char *str
, const struct kernel_param
*kp
)
92 return null_param_store_val(str
, &queue_mode
, NULL_Q_BIO
, NULL_Q_MQ
);
95 static const struct kernel_param_ops null_queue_mode_param_ops
= {
96 .set
= null_set_queue_mode
,
100 device_param_cb(queue_mode
, &null_queue_mode_param_ops
, &queue_mode
, S_IRUGO
);
101 MODULE_PARM_DESC(queue_mode
, "Block interface to use (0=bio,1=rq,2=multiqueue)");
104 module_param(gb
, int, S_IRUGO
);
105 MODULE_PARM_DESC(gb
, "Size in GB");
108 module_param(bs
, int, S_IRUGO
);
109 MODULE_PARM_DESC(bs
, "Block size (in bytes)");
111 static int nr_devices
= 2;
112 module_param(nr_devices
, int, S_IRUGO
);
113 MODULE_PARM_DESC(nr_devices
, "Number of devices to register");
115 static bool use_lightnvm
;
116 module_param(use_lightnvm
, bool, S_IRUGO
);
117 MODULE_PARM_DESC(use_lightnvm
, "Register as a LightNVM device");
119 static int irqmode
= NULL_IRQ_SOFTIRQ
;
121 static int null_set_irqmode(const char *str
, const struct kernel_param
*kp
)
123 return null_param_store_val(str
, &irqmode
, NULL_IRQ_NONE
,
127 static const struct kernel_param_ops null_irqmode_param_ops
= {
128 .set
= null_set_irqmode
,
129 .get
= param_get_int
,
132 device_param_cb(irqmode
, &null_irqmode_param_ops
, &irqmode
, S_IRUGO
);
133 MODULE_PARM_DESC(irqmode
, "IRQ completion handler. 0-none, 1-softirq, 2-timer");
135 static unsigned long completion_nsec
= 10000;
136 module_param(completion_nsec
, ulong
, S_IRUGO
);
137 MODULE_PARM_DESC(completion_nsec
, "Time in ns to complete a request in hardware. Default: 10,000ns");
139 static int hw_queue_depth
= 64;
140 module_param(hw_queue_depth
, int, S_IRUGO
);
141 MODULE_PARM_DESC(hw_queue_depth
, "Queue depth for each hardware queue. Default: 64");
143 static bool use_per_node_hctx
= false;
144 module_param(use_per_node_hctx
, bool, S_IRUGO
);
145 MODULE_PARM_DESC(use_per_node_hctx
, "Use per-node allocation for hardware context queues. Default: false");
147 static void put_tag(struct nullb_queue
*nq
, unsigned int tag
)
149 clear_bit_unlock(tag
, nq
->tag_map
);
151 if (waitqueue_active(&nq
->wait
))
155 static unsigned int get_tag(struct nullb_queue
*nq
)
160 tag
= find_first_zero_bit(nq
->tag_map
, nq
->queue_depth
);
161 if (tag
>= nq
->queue_depth
)
163 } while (test_and_set_bit_lock(tag
, nq
->tag_map
));
168 static void free_cmd(struct nullb_cmd
*cmd
)
170 put_tag(cmd
->nq
, cmd
->tag
);
173 static enum hrtimer_restart
null_cmd_timer_expired(struct hrtimer
*timer
);
175 static struct nullb_cmd
*__alloc_cmd(struct nullb_queue
*nq
)
177 struct nullb_cmd
*cmd
;
182 cmd
= &nq
->cmds
[tag
];
185 if (irqmode
== NULL_IRQ_TIMER
) {
186 hrtimer_init(&cmd
->timer
, CLOCK_MONOTONIC
,
188 cmd
->timer
.function
= null_cmd_timer_expired
;
196 static struct nullb_cmd
*alloc_cmd(struct nullb_queue
*nq
, int can_wait
)
198 struct nullb_cmd
*cmd
;
201 cmd
= __alloc_cmd(nq
);
202 if (cmd
|| !can_wait
)
206 prepare_to_wait(&nq
->wait
, &wait
, TASK_UNINTERRUPTIBLE
);
207 cmd
= __alloc_cmd(nq
);
214 finish_wait(&nq
->wait
, &wait
);
218 static void end_cmd(struct nullb_cmd
*cmd
)
220 struct request_queue
*q
= NULL
;
222 switch (queue_mode
) {
224 blk_mq_end_request(cmd
->rq
, 0);
227 INIT_LIST_HEAD(&cmd
->rq
->queuelist
);
228 blk_end_request_all(cmd
->rq
, 0);
238 /* Restart queue if needed, as we are freeing a tag */
239 if (q
&& !q
->mq_ops
&& blk_queue_stopped(q
)) {
242 spin_lock_irqsave(q
->queue_lock
, flags
);
243 if (blk_queue_stopped(q
))
245 spin_unlock_irqrestore(q
->queue_lock
, flags
);
251 static enum hrtimer_restart
null_cmd_timer_expired(struct hrtimer
*timer
)
253 end_cmd(container_of(timer
, struct nullb_cmd
, timer
));
255 return HRTIMER_NORESTART
;
258 static void null_cmd_end_timer(struct nullb_cmd
*cmd
)
260 ktime_t kt
= ktime_set(0, completion_nsec
);
262 hrtimer_start(&cmd
->timer
, kt
, HRTIMER_MODE_REL
);
265 static void null_softirq_done_fn(struct request
*rq
)
267 if (queue_mode
== NULL_Q_MQ
)
268 end_cmd(blk_mq_rq_to_pdu(rq
));
270 end_cmd(rq
->special
);
273 static inline void null_handle_cmd(struct nullb_cmd
*cmd
)
275 /* Complete IO by inline, softirq or timer */
277 case NULL_IRQ_SOFTIRQ
:
278 switch (queue_mode
) {
280 blk_mq_complete_request(cmd
->rq
, cmd
->rq
->errors
);
283 blk_complete_request(cmd
->rq
);
287 * XXX: no proper submitting cpu information available.
297 null_cmd_end_timer(cmd
);
302 static struct nullb_queue
*nullb_to_queue(struct nullb
*nullb
)
306 if (nullb
->nr_queues
!= 1)
307 index
= raw_smp_processor_id() / ((nr_cpu_ids
+ nullb
->nr_queues
- 1) / nullb
->nr_queues
);
309 return &nullb
->queues
[index
];
312 static blk_qc_t
null_queue_bio(struct request_queue
*q
, struct bio
*bio
)
314 struct nullb
*nullb
= q
->queuedata
;
315 struct nullb_queue
*nq
= nullb_to_queue(nullb
);
316 struct nullb_cmd
*cmd
;
318 cmd
= alloc_cmd(nq
, 1);
321 null_handle_cmd(cmd
);
322 return BLK_QC_T_NONE
;
325 static int null_rq_prep_fn(struct request_queue
*q
, struct request
*req
)
327 struct nullb
*nullb
= q
->queuedata
;
328 struct nullb_queue
*nq
= nullb_to_queue(nullb
);
329 struct nullb_cmd
*cmd
;
331 cmd
= alloc_cmd(nq
, 0);
339 return BLKPREP_DEFER
;
342 static void null_request_fn(struct request_queue
*q
)
346 while ((rq
= blk_fetch_request(q
)) != NULL
) {
347 struct nullb_cmd
*cmd
= rq
->special
;
349 spin_unlock_irq(q
->queue_lock
);
350 null_handle_cmd(cmd
);
351 spin_lock_irq(q
->queue_lock
);
355 static int null_queue_rq(struct blk_mq_hw_ctx
*hctx
,
356 const struct blk_mq_queue_data
*bd
)
358 struct nullb_cmd
*cmd
= blk_mq_rq_to_pdu(bd
->rq
);
360 if (irqmode
== NULL_IRQ_TIMER
) {
361 hrtimer_init(&cmd
->timer
, CLOCK_MONOTONIC
, HRTIMER_MODE_REL
);
362 cmd
->timer
.function
= null_cmd_timer_expired
;
365 cmd
->nq
= hctx
->driver_data
;
367 blk_mq_start_request(bd
->rq
);
369 null_handle_cmd(cmd
);
370 return BLK_MQ_RQ_QUEUE_OK
;
373 static void null_init_queue(struct nullb
*nullb
, struct nullb_queue
*nq
)
378 init_waitqueue_head(&nq
->wait
);
379 nq
->queue_depth
= nullb
->queue_depth
;
382 static int null_init_hctx(struct blk_mq_hw_ctx
*hctx
, void *data
,
385 struct nullb
*nullb
= data
;
386 struct nullb_queue
*nq
= &nullb
->queues
[index
];
388 hctx
->driver_data
= nq
;
389 null_init_queue(nullb
, nq
);
395 static struct blk_mq_ops null_mq_ops
= {
396 .queue_rq
= null_queue_rq
,
397 .map_queue
= blk_mq_map_queue
,
398 .init_hctx
= null_init_hctx
,
399 .complete
= null_softirq_done_fn
,
402 static void cleanup_queue(struct nullb_queue
*nq
)
408 static void cleanup_queues(struct nullb
*nullb
)
412 for (i
= 0; i
< nullb
->nr_queues
; i
++)
413 cleanup_queue(&nullb
->queues
[i
]);
415 kfree(nullb
->queues
);
418 static void null_del_dev(struct nullb
*nullb
)
420 list_del_init(&nullb
->list
);
423 nvm_unregister(nullb
->disk_name
);
425 del_gendisk(nullb
->disk
);
426 blk_cleanup_queue(nullb
->q
);
427 if (queue_mode
== NULL_Q_MQ
)
428 blk_mq_free_tag_set(&nullb
->tag_set
);
430 put_disk(nullb
->disk
);
431 cleanup_queues(nullb
);
437 static void null_lnvm_end_io(struct request
*rq
, int error
)
439 struct nvm_rq
*rqd
= rq
->end_io_data
;
440 struct nvm_dev
*dev
= rqd
->dev
;
442 dev
->mt
->end_io(rqd
, error
);
447 static int null_lnvm_submit_io(struct nvm_dev
*dev
, struct nvm_rq
*rqd
)
449 struct request_queue
*q
= dev
->q
;
451 struct bio
*bio
= rqd
->bio
;
453 rq
= blk_mq_alloc_request(q
, bio_rw(bio
), GFP_KERNEL
, 0);
457 rq
->cmd_type
= REQ_TYPE_DRV_PRIV
;
458 rq
->__sector
= bio
->bi_iter
.bi_sector
;
459 rq
->ioprio
= bio_prio(bio
);
461 if (bio_has_data(bio
))
462 rq
->nr_phys_segments
= bio_phys_segments(q
, bio
);
464 rq
->__data_len
= bio
->bi_iter
.bi_size
;
465 rq
->bio
= rq
->biotail
= bio
;
467 rq
->end_io_data
= rqd
;
469 blk_execute_rq_nowait(q
, NULL
, rq
, 0, null_lnvm_end_io
);
474 static int null_lnvm_id(struct nvm_dev
*dev
, struct nvm_id
*id
)
476 sector_t size
= gb
* 1024 * 1024 * 1024ULL;
478 struct nvm_id_group
*grp
;
486 id
->ppaf
.blk_offset
= 0;
487 id
->ppaf
.blk_len
= 16;
488 id
->ppaf
.pg_offset
= 16;
489 id
->ppaf
.pg_len
= 16;
490 id
->ppaf
.sect_offset
= 32;
491 id
->ppaf
.sect_len
= 8;
492 id
->ppaf
.pln_offset
= 40;
493 id
->ppaf
.pln_len
= 8;
494 id
->ppaf
.lun_offset
= 48;
495 id
->ppaf
.lun_len
= 8;
496 id
->ppaf
.ch_offset
= 56;
499 do_div(size
, bs
); /* convert size to pages */
500 do_div(size
, 256); /* concert size to pgs pr blk */
501 grp
= &id
->groups
[0];
507 do_div(size
, (1 << 16));
508 grp
->num_lun
= size
+ 1;
509 do_div(blksize
, grp
->num_lun
);
510 grp
->num_blk
= blksize
;
521 grp
->mpos
= 0x010101; /* single plane rwe */
522 grp
->cpar
= hw_queue_depth
;
527 static void *null_lnvm_create_dma_pool(struct nvm_dev
*dev
, char *name
)
529 mempool_t
*virtmem_pool
;
531 virtmem_pool
= mempool_create_slab_pool(64, ppa_cache
);
533 pr_err("null_blk: Unable to create virtual memory pool\n");
540 static void null_lnvm_destroy_dma_pool(void *pool
)
542 mempool_destroy(pool
);
545 static void *null_lnvm_dev_dma_alloc(struct nvm_dev
*dev
, void *pool
,
546 gfp_t mem_flags
, dma_addr_t
*dma_handler
)
548 return mempool_alloc(pool
, mem_flags
);
551 static void null_lnvm_dev_dma_free(void *pool
, void *entry
,
552 dma_addr_t dma_handler
)
554 mempool_free(entry
, pool
);
557 static struct nvm_dev_ops null_lnvm_dev_ops
= {
558 .identity
= null_lnvm_id
,
559 .submit_io
= null_lnvm_submit_io
,
561 .create_dma_pool
= null_lnvm_create_dma_pool
,
562 .destroy_dma_pool
= null_lnvm_destroy_dma_pool
,
563 .dev_dma_alloc
= null_lnvm_dev_dma_alloc
,
564 .dev_dma_free
= null_lnvm_dev_dma_free
,
566 /* Simulate nvme protocol restriction */
570 static struct nvm_dev_ops null_lnvm_dev_ops
;
571 #endif /* CONFIG_NVM */
573 static int null_open(struct block_device
*bdev
, fmode_t mode
)
578 static void null_release(struct gendisk
*disk
, fmode_t mode
)
582 static const struct block_device_operations null_fops
= {
583 .owner
= THIS_MODULE
,
585 .release
= null_release
,
588 static int setup_commands(struct nullb_queue
*nq
)
590 struct nullb_cmd
*cmd
;
593 nq
->cmds
= kzalloc(nq
->queue_depth
* sizeof(*cmd
), GFP_KERNEL
);
597 tag_size
= ALIGN(nq
->queue_depth
, BITS_PER_LONG
) / BITS_PER_LONG
;
598 nq
->tag_map
= kzalloc(tag_size
* sizeof(unsigned long), GFP_KERNEL
);
604 for (i
= 0; i
< nq
->queue_depth
; i
++) {
606 INIT_LIST_HEAD(&cmd
->list
);
607 cmd
->ll_list
.next
= NULL
;
614 static int setup_queues(struct nullb
*nullb
)
616 nullb
->queues
= kzalloc(submit_queues
* sizeof(struct nullb_queue
),
621 nullb
->nr_queues
= 0;
622 nullb
->queue_depth
= hw_queue_depth
;
627 static int init_driver_queues(struct nullb
*nullb
)
629 struct nullb_queue
*nq
;
632 for (i
= 0; i
< submit_queues
; i
++) {
633 nq
= &nullb
->queues
[i
];
635 null_init_queue(nullb
, nq
);
637 ret
= setup_commands(nq
);
645 static int null_add_dev(void)
647 struct gendisk
*disk
;
652 nullb
= kzalloc_node(sizeof(*nullb
), GFP_KERNEL
, home_node
);
658 spin_lock_init(&nullb
->lock
);
660 if (queue_mode
== NULL_Q_MQ
&& use_per_node_hctx
)
661 submit_queues
= nr_online_nodes
;
663 rv
= setup_queues(nullb
);
667 if (queue_mode
== NULL_Q_MQ
) {
668 nullb
->tag_set
.ops
= &null_mq_ops
;
669 nullb
->tag_set
.nr_hw_queues
= submit_queues
;
670 nullb
->tag_set
.queue_depth
= hw_queue_depth
;
671 nullb
->tag_set
.numa_node
= home_node
;
672 nullb
->tag_set
.cmd_size
= sizeof(struct nullb_cmd
);
673 nullb
->tag_set
.flags
= BLK_MQ_F_SHOULD_MERGE
;
674 nullb
->tag_set
.driver_data
= nullb
;
676 rv
= blk_mq_alloc_tag_set(&nullb
->tag_set
);
678 goto out_cleanup_queues
;
680 nullb
->q
= blk_mq_init_queue(&nullb
->tag_set
);
681 if (IS_ERR(nullb
->q
)) {
683 goto out_cleanup_tags
;
685 } else if (queue_mode
== NULL_Q_BIO
) {
686 nullb
->q
= blk_alloc_queue_node(GFP_KERNEL
, home_node
);
689 goto out_cleanup_queues
;
691 blk_queue_make_request(nullb
->q
, null_queue_bio
);
692 rv
= init_driver_queues(nullb
);
694 goto out_cleanup_blk_queue
;
696 nullb
->q
= blk_init_queue_node(null_request_fn
, &nullb
->lock
, home_node
);
699 goto out_cleanup_queues
;
701 blk_queue_prep_rq(nullb
->q
, null_rq_prep_fn
);
702 blk_queue_softirq_done(nullb
->q
, null_softirq_done_fn
);
703 rv
= init_driver_queues(nullb
);
705 goto out_cleanup_blk_queue
;
708 nullb
->q
->queuedata
= nullb
;
709 queue_flag_set_unlocked(QUEUE_FLAG_NONROT
, nullb
->q
);
710 queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM
, nullb
->q
);
714 list_add_tail(&nullb
->list
, &nullb_list
);
715 nullb
->index
= nullb_indexes
++;
718 blk_queue_logical_block_size(nullb
->q
, bs
);
719 blk_queue_physical_block_size(nullb
->q
, bs
);
721 sprintf(nullb
->disk_name
, "nullb%d", nullb
->index
);
724 rv
= nvm_register(nullb
->q
, nullb
->disk_name
,
727 goto out_cleanup_blk_queue
;
731 disk
= nullb
->disk
= alloc_disk_node(1, home_node
);
734 goto out_cleanup_lightnvm
;
736 size
= gb
* 1024 * 1024 * 1024ULL;
737 set_capacity(disk
, size
>> 9);
739 disk
->flags
|= GENHD_FL_EXT_DEVT
| GENHD_FL_SUPPRESS_PARTITION_INFO
;
740 disk
->major
= null_major
;
741 disk
->first_minor
= nullb
->index
;
742 disk
->fops
= &null_fops
;
743 disk
->private_data
= nullb
;
744 disk
->queue
= nullb
->q
;
745 strncpy(disk
->disk_name
, nullb
->disk_name
, DISK_NAME_LEN
);
751 out_cleanup_lightnvm
:
753 nvm_unregister(nullb
->disk_name
);
754 out_cleanup_blk_queue
:
755 blk_cleanup_queue(nullb
->q
);
757 if (queue_mode
== NULL_Q_MQ
)
758 blk_mq_free_tag_set(&nullb
->tag_set
);
760 cleanup_queues(nullb
);
767 static int __init
null_init(void)
773 if (bs
> PAGE_SIZE
) {
774 pr_warn("null_blk: invalid block size\n");
775 pr_warn("null_blk: defaults block size to %lu\n", PAGE_SIZE
);
779 if (use_lightnvm
&& bs
!= 4096) {
780 pr_warn("null_blk: LightNVM only supports 4k block size\n");
781 pr_warn("null_blk: defaults block size to 4k\n");
785 if (use_lightnvm
&& queue_mode
!= NULL_Q_MQ
) {
786 pr_warn("null_blk: LightNVM only supported for blk-mq\n");
787 pr_warn("null_blk: defaults queue mode to blk-mq\n");
788 queue_mode
= NULL_Q_MQ
;
791 if (queue_mode
== NULL_Q_MQ
&& use_per_node_hctx
) {
792 if (submit_queues
< nr_online_nodes
) {
793 pr_warn("null_blk: submit_queues param is set to %u.",
795 submit_queues
= nr_online_nodes
;
797 } else if (submit_queues
> nr_cpu_ids
)
798 submit_queues
= nr_cpu_ids
;
799 else if (!submit_queues
)
804 null_major
= register_blkdev(0, "nullb");
809 ppa_cache
= kmem_cache_create("ppa_cache", 64 * sizeof(u64
),
812 pr_err("null_blk: unable to create ppa cache\n");
818 for (i
= 0; i
< nr_devices
; i
++) {
819 ret
= null_add_dev();
824 pr_info("null: module loaded\n");
828 while (!list_empty(&nullb_list
)) {
829 nullb
= list_entry(nullb_list
.next
, struct nullb
, list
);
832 kmem_cache_destroy(ppa_cache
);
834 unregister_blkdev(null_major
, "nullb");
838 static void __exit
null_exit(void)
842 unregister_blkdev(null_major
, "nullb");
845 while (!list_empty(&nullb_list
)) {
846 nullb
= list_entry(nullb_list
.next
, struct nullb
, list
);
851 kmem_cache_destroy(ppa_cache
);
854 module_init(null_init
);
855 module_exit(null_exit
);
857 MODULE_AUTHOR("Jens Axboe <jaxboe@fusionio.com>");
858 MODULE_LICENSE("GPL");