Commit | Line | Data |
---|---|---|
86db1e29 JA |
1 | /* |
2 | * Functions related to barrier IO handling | |
3 | */ | |
4 | #include <linux/kernel.h> | |
5 | #include <linux/module.h> | |
6 | #include <linux/bio.h> | |
7 | #include <linux/blkdev.h> | |
5a0e3ad6 | 8 | #include <linux/gfp.h> |
86db1e29 JA |
9 | |
10 | #include "blk.h" | |
11 | ||
dd4c133f | 12 | static struct request *queue_next_fseq(struct request_queue *q); |
28e7d184 | 13 | |
dd4c133f | 14 | unsigned blk_flush_cur_seq(struct request_queue *q) |
86db1e29 | 15 | { |
dd4c133f | 16 | if (!q->flush_seq) |
86db1e29 | 17 | return 0; |
dd4c133f | 18 | return 1 << ffz(q->flush_seq); |
86db1e29 JA |
19 | } |
20 | ||
dd4c133f TH |
21 | static struct request *blk_flush_complete_seq(struct request_queue *q, |
22 | unsigned seq, int error) | |
86db1e29 | 23 | { |
28e7d184 | 24 | struct request *next_rq = NULL; |
86db1e29 | 25 | |
dd4c133f TH |
26 | if (error && !q->flush_err) |
27 | q->flush_err = error; | |
86db1e29 | 28 | |
dd4c133f TH |
29 | BUG_ON(q->flush_seq & seq); |
30 | q->flush_seq |= seq; | |
86db1e29 | 31 | |
dd4c133f TH |
32 | if (blk_flush_cur_seq(q) != QUEUE_FSEQ_DONE) { |
33 | /* not complete yet, queue the next flush sequence */ | |
34 | next_rq = queue_next_fseq(q); | |
28e7d184 | 35 | } else { |
dd4c133f TH |
36 | /* complete this flush request */ |
37 | __blk_end_request_all(q->orig_flush_rq, q->flush_err); | |
38 | q->orig_flush_rq = NULL; | |
39 | q->flush_seq = 0; | |
40 | ||
41 | /* dispatch the next flush if there's one */ | |
42 | if (!list_empty(&q->pending_flushes)) { | |
43 | next_rq = list_entry_rq(q->pending_flushes.next); | |
28e7d184 TH |
44 | list_move(&next_rq->queuelist, &q->queue_head); |
45 | } | |
46 | } | |
47 | return next_rq; | |
86db1e29 JA |
48 | } |
49 | ||
50 | static void pre_flush_end_io(struct request *rq, int error) | |
51 | { | |
52 | elv_completed_request(rq->q, rq); | |
dd4c133f | 53 | blk_flush_complete_seq(rq->q, QUEUE_FSEQ_PREFLUSH, error); |
86db1e29 JA |
54 | } |
55 | ||
dd4c133f | 56 | static void flush_data_end_io(struct request *rq, int error) |
86db1e29 JA |
57 | { |
58 | elv_completed_request(rq->q, rq); | |
dd4c133f | 59 | blk_flush_complete_seq(rq->q, QUEUE_FSEQ_DATA, error); |
86db1e29 JA |
60 | } |
61 | ||
62 | static void post_flush_end_io(struct request *rq, int error) | |
63 | { | |
64 | elv_completed_request(rq->q, rq); | |
dd4c133f | 65 | blk_flush_complete_seq(rq->q, QUEUE_FSEQ_POSTFLUSH, error); |
86db1e29 JA |
66 | } |
67 | ||
28e7d184 TH |
68 | static void queue_flush(struct request_queue *q, struct request *rq, |
69 | rq_end_io_fn *end_io) | |
86db1e29 | 70 | { |
2a4aa30c | 71 | blk_rq_init(q, rq); |
28e18d01 | 72 | rq->cmd_type = REQ_TYPE_FS; |
28e7d184 | 73 | rq->cmd_flags = REQ_FLUSH; |
dd4c133f | 74 | rq->rq_disk = q->orig_flush_rq->rq_disk; |
86db1e29 | 75 | rq->end_io = end_io; |
86db1e29 JA |
76 | |
77 | elv_insert(q, rq, ELEVATOR_INSERT_FRONT); | |
78 | } | |
79 | ||
dd4c133f | 80 | static struct request *queue_next_fseq(struct request_queue *q) |
86db1e29 | 81 | { |
dd4c133f | 82 | struct request *rq = &q->flush_rq; |
86db1e29 | 83 | |
dd4c133f TH |
84 | switch (blk_flush_cur_seq(q)) { |
85 | case QUEUE_FSEQ_PREFLUSH: | |
28e7d184 TH |
86 | queue_flush(q, rq, pre_flush_end_io); |
87 | break; | |
f671620e | 88 | |
dd4c133f | 89 | case QUEUE_FSEQ_DATA: |
f671620e TH |
90 | /* initialize proxy request and queue it */ |
91 | blk_rq_init(q, rq); | |
dd4c133f | 92 | init_request_from_bio(rq, q->orig_flush_rq->bio); |
28e7d184 | 93 | rq->cmd_flags &= ~REQ_HARDBARRIER; |
f671620e TH |
94 | if (q->ordered & QUEUE_ORDERED_DO_FUA) |
95 | rq->cmd_flags |= REQ_FUA; | |
dd4c133f | 96 | rq->end_io = flush_data_end_io; |
f671620e TH |
97 | |
98 | elv_insert(q, rq, ELEVATOR_INSERT_FRONT); | |
28e7d184 | 99 | break; |
86db1e29 | 100 | |
dd4c133f | 101 | case QUEUE_FSEQ_POSTFLUSH: |
28e7d184 TH |
102 | queue_flush(q, rq, post_flush_end_io); |
103 | break; | |
86db1e29 | 104 | |
28e7d184 TH |
105 | default: |
106 | BUG(); | |
107 | } | |
dd831006 | 108 | return rq; |
86db1e29 JA |
109 | } |
110 | ||
dd4c133f | 111 | struct request *blk_do_flush(struct request_queue *q, struct request *rq) |
86db1e29 | 112 | { |
28e7d184 TH |
113 | unsigned skip = 0; |
114 | ||
115 | if (!(rq->cmd_flags & REQ_HARDBARRIER)) | |
116 | return rq; | |
117 | ||
dd4c133f | 118 | if (q->flush_seq) { |
28e7d184 | 119 | /* |
dd4c133f TH |
120 | * Sequenced flush is already in progress and they |
121 | * can't be processed in parallel. Queue for later | |
122 | * processing. | |
28e7d184 | 123 | */ |
dd4c133f | 124 | list_move_tail(&rq->queuelist, &q->pending_flushes); |
28e7d184 TH |
125 | return NULL; |
126 | } | |
127 | ||
128 | if (unlikely(q->next_ordered == QUEUE_ORDERED_NONE)) { | |
129 | /* | |
130 | * Queue ordering not supported. Terminate | |
131 | * with prejudice. | |
132 | */ | |
133 | blk_dequeue_request(rq); | |
134 | __blk_end_request_all(rq, -EOPNOTSUPP); | |
135 | return NULL; | |
86db1e29 JA |
136 | } |
137 | ||
138 | /* | |
dd4c133f | 139 | * Start a new flush sequence |
86db1e29 | 140 | */ |
dd4c133f | 141 | q->flush_err = 0; |
28e7d184 | 142 | q->ordered = q->next_ordered; |
dd4c133f | 143 | q->flush_seq |= QUEUE_FSEQ_STARTED; |
86db1e29 | 144 | |
28e7d184 TH |
145 | /* |
146 | * For an empty barrier, there's no actual BAR request, which | |
147 | * in turn makes POSTFLUSH unnecessary. Mask them off. | |
148 | */ | |
149 | if (!blk_rq_sectors(rq)) | |
150 | q->ordered &= ~(QUEUE_ORDERED_DO_BAR | | |
151 | QUEUE_ORDERED_DO_POSTFLUSH); | |
152 | ||
153 | /* stash away the original request */ | |
154 | blk_dequeue_request(rq); | |
dd4c133f | 155 | q->orig_flush_rq = rq; |
86db1e29 | 156 | |
28e7d184 | 157 | if (!(q->ordered & QUEUE_ORDERED_DO_PREFLUSH)) |
dd4c133f | 158 | skip |= QUEUE_FSEQ_PREFLUSH; |
86db1e29 | 159 | |
28e7d184 | 160 | if (!(q->ordered & QUEUE_ORDERED_DO_BAR)) |
dd4c133f | 161 | skip |= QUEUE_FSEQ_DATA; |
28e7d184 TH |
162 | |
163 | if (!(q->ordered & QUEUE_ORDERED_DO_POSTFLUSH)) | |
dd4c133f | 164 | skip |= QUEUE_FSEQ_POSTFLUSH; |
28e7d184 TH |
165 | |
166 | /* complete skipped sequences and return the first sequence */ | |
dd4c133f | 167 | return blk_flush_complete_seq(q, skip, 0); |
86db1e29 JA |
168 | } |
169 | ||
170 | static void bio_end_empty_barrier(struct bio *bio, int err) | |
171 | { | |
cc66b451 JA |
172 | if (err) { |
173 | if (err == -EOPNOTSUPP) | |
174 | set_bit(BIO_EOPNOTSUPP, &bio->bi_flags); | |
86db1e29 | 175 | clear_bit(BIO_UPTODATE, &bio->bi_flags); |
cc66b451 | 176 | } |
f17e232e DM |
177 | if (bio->bi_private) |
178 | complete(bio->bi_private); | |
179 | bio_put(bio); | |
86db1e29 JA |
180 | } |
181 | ||
182 | /** | |
183 | * blkdev_issue_flush - queue a flush | |
184 | * @bdev: blockdev to issue flush for | |
fbd9b09a | 185 | * @gfp_mask: memory allocation flags (for bio_alloc) |
86db1e29 | 186 | * @error_sector: error sector |
fbd9b09a | 187 | * @flags: BLKDEV_IFL_* flags to control behaviour |
86db1e29 JA |
188 | * |
189 | * Description: | |
190 | * Issue a flush for the block device in question. Caller can supply | |
191 | * room for storing the error offset in case of a flush error, if they | |
f17e232e DM |
192 | * wish to. If WAIT flag is not passed then caller may check only what |
193 | * request was pushed in some internal queue for later handling. | |
86db1e29 | 194 | */ |
fbd9b09a DM |
195 | int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask, |
196 | sector_t *error_sector, unsigned long flags) | |
86db1e29 JA |
197 | { |
198 | DECLARE_COMPLETION_ONSTACK(wait); | |
199 | struct request_queue *q; | |
200 | struct bio *bio; | |
fbd9b09a | 201 | int ret = 0; |
86db1e29 JA |
202 | |
203 | if (bdev->bd_disk == NULL) | |
204 | return -ENXIO; | |
205 | ||
206 | q = bdev_get_queue(bdev); | |
207 | if (!q) | |
208 | return -ENXIO; | |
209 | ||
f10d9f61 DC |
210 | /* |
211 | * some block devices may not have their queue correctly set up here | |
212 | * (e.g. loop device without a backing file) and so issuing a flush | |
213 | * here will panic. Ensure there is a request function before issuing | |
214 | * the barrier. | |
215 | */ | |
216 | if (!q->make_request_fn) | |
217 | return -ENXIO; | |
218 | ||
fbd9b09a | 219 | bio = bio_alloc(gfp_mask, 0); |
86db1e29 | 220 | bio->bi_end_io = bio_end_empty_barrier; |
86db1e29 | 221 | bio->bi_bdev = bdev; |
f17e232e DM |
222 | if (test_bit(BLKDEV_WAIT, &flags)) |
223 | bio->bi_private = &wait; | |
86db1e29 | 224 | |
f17e232e DM |
225 | bio_get(bio); |
226 | submit_bio(WRITE_BARRIER, bio); | |
227 | if (test_bit(BLKDEV_WAIT, &flags)) { | |
228 | wait_for_completion(&wait); | |
229 | /* | |
230 | * The driver must store the error location in ->bi_sector, if | |
231 | * it supports it. For non-stacked drivers, this should be | |
232 | * copied from blk_rq_pos(rq). | |
233 | */ | |
234 | if (error_sector) | |
235 | *error_sector = bio->bi_sector; | |
236 | } | |
86db1e29 | 237 | |
cc66b451 JA |
238 | if (bio_flagged(bio, BIO_EOPNOTSUPP)) |
239 | ret = -EOPNOTSUPP; | |
240 | else if (!bio_flagged(bio, BIO_UPTODATE)) | |
86db1e29 JA |
241 | ret = -EIO; |
242 | ||
243 | bio_put(bio); | |
244 | return ret; | |
245 | } | |
86db1e29 | 246 | EXPORT_SYMBOL(blkdev_issue_flush); |