Commit | Line | Data |
---|---|---|
320ae51f JA |
1 | #ifndef BLK_MQ_H |
2 | #define BLK_MQ_H | |
3 | ||
4 | #include <linux/blkdev.h> | |
5 | ||
6 | struct blk_mq_tags; | |
7 | ||
8 | struct blk_mq_cpu_notifier { | |
9 | struct list_head list; | |
10 | void *data; | |
e814e71b | 11 | int (*notify)(void *data, unsigned long action, unsigned int cpu); |
320ae51f JA |
12 | }; |
13 | ||
1429d7c9 JA |
14 | struct blk_mq_ctxmap { |
15 | unsigned int map_size; | |
16 | unsigned int bits_per_word; | |
17 | struct blk_align_bitmap *map; | |
18 | }; | |
19 | ||
320ae51f JA |
20 | struct blk_mq_hw_ctx { |
21 | struct { | |
22 | spinlock_t lock; | |
23 | struct list_head dispatch; | |
24 | } ____cacheline_aligned_in_smp; | |
25 | ||
26 | unsigned long state; /* BLK_MQ_S_* flags */ | |
70f4db63 CH |
27 | struct delayed_work run_work; |
28 | struct delayed_work delay_work; | |
e4043dcf | 29 | cpumask_var_t cpumask; |
506e931f JA |
30 | int next_cpu; |
31 | int next_cpu_batch; | |
320ae51f JA |
32 | |
33 | unsigned long flags; /* BLK_MQ_F_* flags */ | |
34 | ||
35 | struct request_queue *queue; | |
36 | unsigned int queue_num; | |
37 | ||
38 | void *driver_data; | |
39 | ||
1429d7c9 JA |
40 | struct blk_mq_ctxmap ctx_map; |
41 | ||
4bb659b1 JA |
42 | unsigned int nr_ctx; |
43 | struct blk_mq_ctx **ctxs; | |
44 | ||
45 | unsigned int wait_index; | |
320ae51f | 46 | |
320ae51f JA |
47 | struct blk_mq_tags *tags; |
48 | ||
49 | unsigned long queued; | |
50 | unsigned long run; | |
51 | #define BLK_MQ_MAX_DISPATCH_ORDER 10 | |
52 | unsigned long dispatched[BLK_MQ_MAX_DISPATCH_ORDER]; | |
53 | ||
320ae51f JA |
54 | unsigned int numa_node; |
55 | unsigned int cmd_size; /* per-request extra data */ | |
56 | ||
0d2602ca JA |
57 | atomic_t nr_active; |
58 | ||
320ae51f JA |
59 | struct blk_mq_cpu_notifier cpu_notifier; |
60 | struct kobject kobj; | |
61 | }; | |
62 | ||
24d2f903 | 63 | struct blk_mq_tag_set { |
320ae51f JA |
64 | struct blk_mq_ops *ops; |
65 | unsigned int nr_hw_queues; | |
e3a2b3f9 | 66 | unsigned int queue_depth; /* max hw supported */ |
320ae51f JA |
67 | unsigned int reserved_tags; |
68 | unsigned int cmd_size; /* per-request extra data */ | |
69 | int numa_node; | |
70 | unsigned int timeout; | |
71 | unsigned int flags; /* BLK_MQ_F_* */ | |
24d2f903 CH |
72 | void *driver_data; |
73 | ||
74 | struct blk_mq_tags **tags; | |
0d2602ca JA |
75 | |
76 | struct mutex tag_list_lock; | |
77 | struct list_head tag_list; | |
320ae51f JA |
78 | }; |
79 | ||
80 | typedef int (queue_rq_fn)(struct blk_mq_hw_ctx *, struct request *); | |
81 | typedef struct blk_mq_hw_ctx *(map_queue_fn)(struct request_queue *, const int); | |
24d2f903 | 82 | typedef struct blk_mq_hw_ctx *(alloc_hctx_fn)(struct blk_mq_tag_set *, |
f14bbe77 | 83 | unsigned int, int); |
320ae51f JA |
84 | typedef void (free_hctx_fn)(struct blk_mq_hw_ctx *, unsigned int); |
85 | typedef int (init_hctx_fn)(struct blk_mq_hw_ctx *, void *, unsigned int); | |
86 | typedef void (exit_hctx_fn)(struct blk_mq_hw_ctx *, unsigned int); | |
24d2f903 CH |
87 | typedef int (init_request_fn)(void *, struct request *, unsigned int, |
88 | unsigned int, unsigned int); | |
89 | typedef void (exit_request_fn)(void *, struct request *, unsigned int, | |
90 | unsigned int); | |
320ae51f JA |
91 | |
92 | struct blk_mq_ops { | |
93 | /* | |
94 | * Queue request | |
95 | */ | |
96 | queue_rq_fn *queue_rq; | |
97 | ||
98 | /* | |
99 | * Map to specific hardware queue | |
100 | */ | |
101 | map_queue_fn *map_queue; | |
102 | ||
103 | /* | |
104 | * Called on request timeout | |
105 | */ | |
106 | rq_timed_out_fn *timeout; | |
107 | ||
30a91cb4 CH |
108 | softirq_done_fn *complete; |
109 | ||
320ae51f JA |
110 | /* |
111 | * Override for hctx allocations (should probably go) | |
112 | */ | |
113 | alloc_hctx_fn *alloc_hctx; | |
114 | free_hctx_fn *free_hctx; | |
115 | ||
116 | /* | |
117 | * Called when the block layer side of a hardware queue has been | |
118 | * set up, allowing the driver to allocate/init matching structures. | |
119 | * Ditto for exit/teardown. | |
120 | */ | |
121 | init_hctx_fn *init_hctx; | |
122 | exit_hctx_fn *exit_hctx; | |
e9b267d9 CH |
123 | |
124 | /* | |
125 | * Called for every command allocated by the block layer to allow | |
126 | * the driver to set up driver specific data. | |
127 | * Ditto for exit/teardown. | |
128 | */ | |
129 | init_request_fn *init_request; | |
130 | exit_request_fn *exit_request; | |
320ae51f JA |
131 | }; |
132 | ||
133 | enum { | |
134 | BLK_MQ_RQ_QUEUE_OK = 0, /* queued fine */ | |
135 | BLK_MQ_RQ_QUEUE_BUSY = 1, /* requeue IO for later */ | |
136 | BLK_MQ_RQ_QUEUE_ERROR = 2, /* end IO with error */ | |
137 | ||
138 | BLK_MQ_F_SHOULD_MERGE = 1 << 0, | |
139 | BLK_MQ_F_SHOULD_SORT = 1 << 1, | |
0d2602ca | 140 | BLK_MQ_F_TAG_SHARED = 1 << 2, |
320ae51f | 141 | |
5d12f905 | 142 | BLK_MQ_S_STOPPED = 0, |
0d2602ca | 143 | BLK_MQ_S_TAG_ACTIVE = 1, |
320ae51f JA |
144 | |
145 | BLK_MQ_MAX_DEPTH = 2048, | |
506e931f JA |
146 | |
147 | BLK_MQ_CPU_WORK_BATCH = 8, | |
320ae51f JA |
148 | }; |
149 | ||
24d2f903 | 150 | struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *); |
320ae51f JA |
151 | int blk_mq_register_disk(struct gendisk *); |
152 | void blk_mq_unregister_disk(struct gendisk *); | |
320ae51f | 153 | |
24d2f903 CH |
154 | int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set); |
155 | void blk_mq_free_tag_set(struct blk_mq_tag_set *set); | |
156 | ||
320ae51f JA |
157 | void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule); |
158 | ||
feb71dae | 159 | void blk_mq_insert_request(struct request *, bool, bool, bool); |
320ae51f JA |
160 | void blk_mq_run_queues(struct request_queue *q, bool async); |
161 | void blk_mq_free_request(struct request *rq); | |
162 | bool blk_mq_can_queue(struct blk_mq_hw_ctx *); | |
18741986 | 163 | struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp); |
320ae51f | 164 | struct request *blk_mq_alloc_reserved_request(struct request_queue *q, int rw, gfp_t gfp); |
24d2f903 | 165 | struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag); |
320ae51f JA |
166 | |
167 | struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *, const int ctx_index); | |
f14bbe77 | 168 | struct blk_mq_hw_ctx *blk_mq_alloc_single_hw_queue(struct blk_mq_tag_set *, unsigned int, int); |
320ae51f JA |
169 | void blk_mq_free_single_hw_queue(struct blk_mq_hw_ctx *, unsigned int); |
170 | ||
63151a44 CH |
171 | void blk_mq_end_io(struct request *rq, int error); |
172 | void __blk_mq_end_io(struct request *rq, int error); | |
320ae51f | 173 | |
ed0791b2 | 174 | void blk_mq_requeue_request(struct request *rq); |
6fca6a61 CH |
175 | void blk_mq_add_to_requeue_list(struct request *rq, bool at_head); |
176 | void blk_mq_kick_requeue_list(struct request_queue *q); | |
30a91cb4 CH |
177 | void blk_mq_complete_request(struct request *rq); |
178 | ||
320ae51f JA |
179 | void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx); |
180 | void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx); | |
280d45f6 | 181 | void blk_mq_stop_hw_queues(struct request_queue *q); |
2f268556 | 182 | void blk_mq_start_hw_queues(struct request_queue *q); |
1b4a3258 | 183 | void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async); |
70f4db63 | 184 | void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs); |
edf866b3 | 185 | void blk_mq_tag_busy_iter(struct blk_mq_tags *tags, void (*fn)(void *data, unsigned long *), void *data); |
320ae51f JA |
186 | |
187 | /* | |
188 | * Driver command data is immediately after the request. So subtract request | |
189 | * size to get back to the original request. | |
190 | */ | |
191 | static inline struct request *blk_mq_rq_from_pdu(void *pdu) | |
192 | { | |
193 | return pdu - sizeof(struct request); | |
194 | } | |
195 | static inline void *blk_mq_rq_to_pdu(struct request *rq) | |
196 | { | |
197 | return (void *) rq + sizeof(*rq); | |
198 | } | |
199 | ||
320ae51f | 200 | #define queue_for_each_hw_ctx(q, hctx, i) \ |
0d0b7d42 JA |
201 | for ((i) = 0; (i) < (q)->nr_hw_queues && \ |
202 | ({ hctx = (q)->queue_hw_ctx[i]; 1; }); (i)++) | |
320ae51f JA |
203 | |
204 | #define queue_for_each_ctx(q, ctx, i) \ | |
0d0b7d42 JA |
205 | for ((i) = 0; (i) < (q)->nr_queues && \ |
206 | ({ ctx = per_cpu_ptr((q)->queue_ctx, (i)); 1; }); (i)++) | |
320ae51f JA |
207 | |
208 | #define hctx_for_each_ctx(hctx, ctx, i) \ | |
0d0b7d42 JA |
209 | for ((i) = 0; (i) < (hctx)->nr_ctx && \ |
210 | ({ ctx = (hctx)->ctxs[(i)]; 1; }); (i)++) | |
320ae51f JA |
211 | |
212 | #define blk_ctx_sum(q, sum) \ | |
213 | ({ \ | |
214 | struct blk_mq_ctx *__x; \ | |
215 | unsigned int __ret = 0, __i; \ | |
216 | \ | |
217 | queue_for_each_ctx((q), __x, __i) \ | |
218 | __ret += sum; \ | |
219 | __ret; \ | |
220 | }) | |
221 | ||
222 | #endif |