2 * Main bcache entry point - handle a read or a write request and decide what to
3 * do with it; the make_request functions are called by the block layer.
5 * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
6 * Copyright 2012 Google, Inc.
13 #include "writeback.h"
15 #include <linux/cgroup.h>
16 #include <linux/module.h>
17 #include <linux/hash.h>
18 #include <linux/random.h>
19 #include "blk-cgroup.h"
21 #include <trace/events/bcache.h>
23 #define CUTOFF_CACHE_ADD 95
24 #define CUTOFF_CACHE_READA 90
26 struct kmem_cache
*bch_search_cache
;
28 static void bch_data_insert_start(struct closure
*);
30 /* Cgroup interface */
32 #ifdef CONFIG_CGROUP_BCACHE
33 static struct bch_cgroup bcache_default_cgroup
= { .cache_mode
= -1 };
35 static struct bch_cgroup
*cgroup_to_bcache(struct cgroup
*cgroup
)
37 struct cgroup_subsys_state
*css
;
39 (css
= cgroup_subsys_state(cgroup
, bcache_subsys_id
))
40 ? container_of(css
, struct bch_cgroup
, css
)
41 : &bcache_default_cgroup
;
44 struct bch_cgroup
*bch_bio_to_cgroup(struct bio
*bio
)
46 struct cgroup_subsys_state
*css
= bio
->bi_css
47 ? cgroup_subsys_state(bio
->bi_css
->cgroup
, bcache_subsys_id
)
48 : task_subsys_state(current
, bcache_subsys_id
);
51 ? container_of(css
, struct bch_cgroup
, css
)
52 : &bcache_default_cgroup
;
55 static ssize_t
cache_mode_read(struct cgroup
*cgrp
, struct cftype
*cft
,
57 char __user
*buf
, size_t nbytes
, loff_t
*ppos
)
60 int len
= bch_snprint_string_list(tmp
, PAGE_SIZE
, bch_cache_modes
,
61 cgroup_to_bcache(cgrp
)->cache_mode
+ 1);
66 return simple_read_from_buffer(buf
, nbytes
, ppos
, tmp
, len
);
69 static int cache_mode_write(struct cgroup
*cgrp
, struct cftype
*cft
,
72 int v
= bch_read_string_list(buf
, bch_cache_modes
);
76 cgroup_to_bcache(cgrp
)->cache_mode
= v
- 1;
80 static u64
bch_verify_read(struct cgroup
*cgrp
, struct cftype
*cft
)
82 return cgroup_to_bcache(cgrp
)->verify
;
85 static int bch_verify_write(struct cgroup
*cgrp
, struct cftype
*cft
, u64 val
)
87 cgroup_to_bcache(cgrp
)->verify
= val
;
91 static u64
bch_cache_hits_read(struct cgroup
*cgrp
, struct cftype
*cft
)
93 struct bch_cgroup
*bcachecg
= cgroup_to_bcache(cgrp
);
94 return atomic_read(&bcachecg
->stats
.cache_hits
);
97 static u64
bch_cache_misses_read(struct cgroup
*cgrp
, struct cftype
*cft
)
99 struct bch_cgroup
*bcachecg
= cgroup_to_bcache(cgrp
);
100 return atomic_read(&bcachecg
->stats
.cache_misses
);
103 static u64
bch_cache_bypass_hits_read(struct cgroup
*cgrp
,
106 struct bch_cgroup
*bcachecg
= cgroup_to_bcache(cgrp
);
107 return atomic_read(&bcachecg
->stats
.cache_bypass_hits
);
110 static u64
bch_cache_bypass_misses_read(struct cgroup
*cgrp
,
113 struct bch_cgroup
*bcachecg
= cgroup_to_bcache(cgrp
);
114 return atomic_read(&bcachecg
->stats
.cache_bypass_misses
);
117 static struct cftype bch_files
[] = {
119 .name
= "cache_mode",
120 .read
= cache_mode_read
,
121 .write_string
= cache_mode_write
,
125 .read_u64
= bch_verify_read
,
126 .write_u64
= bch_verify_write
,
129 .name
= "cache_hits",
130 .read_u64
= bch_cache_hits_read
,
133 .name
= "cache_misses",
134 .read_u64
= bch_cache_misses_read
,
137 .name
= "cache_bypass_hits",
138 .read_u64
= bch_cache_bypass_hits_read
,
141 .name
= "cache_bypass_misses",
142 .read_u64
= bch_cache_bypass_misses_read
,
147 static void init_bch_cgroup(struct bch_cgroup
*cg
)
152 static struct cgroup_subsys_state
*bcachecg_create(struct cgroup
*cgroup
)
154 struct bch_cgroup
*cg
;
156 cg
= kzalloc(sizeof(*cg
), GFP_KERNEL
);
158 return ERR_PTR(-ENOMEM
);
163 static void bcachecg_destroy(struct cgroup
*cgroup
)
165 struct bch_cgroup
*cg
= cgroup_to_bcache(cgroup
);
166 free_css_id(&bcache_subsys
, &cg
->css
);
170 struct cgroup_subsys bcache_subsys
= {
171 .create
= bcachecg_create
,
172 .destroy
= bcachecg_destroy
,
173 .subsys_id
= bcache_subsys_id
,
175 .module
= THIS_MODULE
,
177 EXPORT_SYMBOL_GPL(bcache_subsys
);
180 static unsigned cache_mode(struct cached_dev
*dc
, struct bio
*bio
)
182 #ifdef CONFIG_CGROUP_BCACHE
183 int r
= bch_bio_to_cgroup(bio
)->cache_mode
;
187 return BDEV_CACHE_MODE(&dc
->sb
);
190 static bool verify(struct cached_dev
*dc
, struct bio
*bio
)
192 #ifdef CONFIG_CGROUP_BCACHE
193 if (bch_bio_to_cgroup(bio
)->verify
)
199 static void bio_csum(struct bio
*bio
, struct bkey
*k
)
205 bio_for_each_segment(bv
, bio
, i
) {
206 void *d
= kmap(bv
->bv_page
) + bv
->bv_offset
;
207 csum
= bch_crc64_update(csum
, d
, bv
->bv_len
);
211 k
->ptr
[KEY_PTRS(k
)] = csum
& (~0ULL >> 1);
214 /* Insert data into cache */
216 static void bch_data_insert_keys(struct closure
*cl
)
218 struct btree_op
*op
= container_of(cl
, struct btree_op
, cl
);
219 struct search
*s
= container_of(op
, struct search
, op
);
222 * If we're looping, might already be waiting on
223 * another journal write - can't wait on more than one journal write at
226 * XXX: this looks wrong
229 while (atomic_read(&s
->cl
.remaining
) & CLOSURE_WAITING
)
230 closure_sync(&s
->cl
);
234 op
->journal
= bch_journal(op
->c
, &s
->insert_keys
,
238 if (bch_btree_insert(op
, op
->c
, &s
->insert_keys
)) {
240 op
->insert_data_done
= true;
244 atomic_dec_bug(op
->journal
);
247 if (!op
->insert_data_done
)
248 continue_at(cl
, bch_data_insert_start
, bcache_wq
);
250 bch_keylist_free(&s
->insert_keys
);
255 struct list_head list
;
256 struct task_struct
*last
;
257 unsigned sectors_free
;
261 void bch_open_buckets_free(struct cache_set
*c
)
263 struct open_bucket
*b
;
265 while (!list_empty(&c
->data_buckets
)) {
266 b
= list_first_entry(&c
->data_buckets
,
267 struct open_bucket
, list
);
273 int bch_open_buckets_alloc(struct cache_set
*c
)
277 spin_lock_init(&c
->data_bucket_lock
);
279 for (i
= 0; i
< 6; i
++) {
280 struct open_bucket
*b
= kzalloc(sizeof(*b
), GFP_KERNEL
);
284 list_add(&b
->list
, &c
->data_buckets
);
291 * We keep multiple buckets open for writes, and try to segregate different
292 * write streams for better cache utilization: first we look for a bucket where
293 * the last write to it was sequential with the current write, and failing that
294 * we look for a bucket that was last used by the same task.
296 * The ideas is if you've got multiple tasks pulling data into the cache at the
297 * same time, you'll get better cache utilization if you try to segregate their
298 * data and preserve locality.
300 * For example, say you've starting Firefox at the same time you're copying a
301 * bunch of files. Firefox will likely end up being fairly hot and stay in the
302 * cache awhile, but the data you copied might not be; if you wrote all that
303 * data to the same buckets it'd get invalidated at the same time.
305 * Both of those tasks will be doing fairly random IO so we can't rely on
306 * detecting sequential IO to segregate their data, but going off of the task
307 * should be a sane heuristic.
309 static struct open_bucket
*pick_data_bucket(struct cache_set
*c
,
310 const struct bkey
*search
,
311 struct task_struct
*task
,
314 struct open_bucket
*ret
, *ret_task
= NULL
;
316 list_for_each_entry_reverse(ret
, &c
->data_buckets
, list
)
317 if (!bkey_cmp(&ret
->key
, search
))
319 else if (ret
->last
== task
)
322 ret
= ret_task
?: list_first_entry(&c
->data_buckets
,
323 struct open_bucket
, list
);
325 if (!ret
->sectors_free
&& KEY_PTRS(alloc
)) {
326 ret
->sectors_free
= c
->sb
.bucket_size
;
327 bkey_copy(&ret
->key
, alloc
);
331 if (!ret
->sectors_free
)
338 * Allocates some space in the cache to write to, and k to point to the newly
339 * allocated space, and updates KEY_SIZE(k) and KEY_OFFSET(k) (to point to the
340 * end of the newly allocated space).
342 * May allocate fewer sectors than @sectors, KEY_SIZE(k) indicates how many
343 * sectors were actually allocated.
345 * If s->writeback is true, will not fail.
347 static bool bch_alloc_sectors(struct bkey
*k
, unsigned sectors
,
350 struct cache_set
*c
= s
->op
.c
;
351 struct open_bucket
*b
;
352 BKEY_PADDED(key
) alloc
;
353 struct closure cl
, *w
= NULL
;
357 closure_init_stack(&cl
);
362 * We might have to allocate a new bucket, which we can't do with a
363 * spinlock held. So if we have to allocate, we drop the lock, allocate
364 * and then retry. KEY_PTRS() indicates whether alloc points to
365 * allocated bucket(s).
368 bkey_init(&alloc
.key
);
369 spin_lock(&c
->data_bucket_lock
);
371 while (!(b
= pick_data_bucket(c
, k
, s
->task
, &alloc
.key
))) {
372 unsigned watermark
= s
->op
.write_prio
376 spin_unlock(&c
->data_bucket_lock
);
378 if (bch_bucket_alloc_set(c
, watermark
, &alloc
.key
, 1, w
))
381 spin_lock(&c
->data_bucket_lock
);
385 * If we had to allocate, we might race and not need to allocate the
386 * second time we call find_data_bucket(). If we allocated a bucket but
387 * didn't use it, drop the refcount bch_bucket_alloc_set() took:
389 if (KEY_PTRS(&alloc
.key
))
390 __bkey_put(c
, &alloc
.key
);
392 for (i
= 0; i
< KEY_PTRS(&b
->key
); i
++)
393 EBUG_ON(ptr_stale(c
, &b
->key
, i
));
395 /* Set up the pointer to the space we're allocating: */
397 for (i
= 0; i
< KEY_PTRS(&b
->key
); i
++)
398 k
->ptr
[i
] = b
->key
.ptr
[i
];
400 sectors
= min(sectors
, b
->sectors_free
);
402 SET_KEY_OFFSET(k
, KEY_OFFSET(k
) + sectors
);
403 SET_KEY_SIZE(k
, sectors
);
404 SET_KEY_PTRS(k
, KEY_PTRS(&b
->key
));
407 * Move b to the end of the lru, and keep track of what this bucket was
410 list_move_tail(&b
->list
, &c
->data_buckets
);
411 bkey_copy_key(&b
->key
, k
);
414 b
->sectors_free
-= sectors
;
416 for (i
= 0; i
< KEY_PTRS(&b
->key
); i
++) {
417 SET_PTR_OFFSET(&b
->key
, i
, PTR_OFFSET(&b
->key
, i
) + sectors
);
419 atomic_long_add(sectors
,
420 &PTR_CACHE(c
, &b
->key
, i
)->sectors_written
);
423 if (b
->sectors_free
< c
->sb
.block_size
)
427 * k takes refcounts on the buckets it points to until it's inserted
428 * into the btree, but if we're done with this bucket we just transfer
429 * get_data_bucket()'s refcount.
432 for (i
= 0; i
< KEY_PTRS(&b
->key
); i
++)
433 atomic_inc(&PTR_BUCKET(c
, &b
->key
, i
)->pin
);
435 spin_unlock(&c
->data_bucket_lock
);
439 static void bch_data_invalidate(struct closure
*cl
)
441 struct btree_op
*op
= container_of(cl
, struct btree_op
, cl
);
442 struct search
*s
= container_of(op
, struct search
, op
);
443 struct bio
*bio
= op
->cache_bio
;
445 pr_debug("invalidating %i sectors from %llu",
446 bio_sectors(bio
), (uint64_t) bio
->bi_sector
);
448 while (bio_sectors(bio
)) {
449 unsigned len
= min(bio_sectors(bio
), 1U << 14);
451 if (bch_keylist_realloc(&s
->insert_keys
, 0, op
->c
))
454 bio
->bi_sector
+= len
;
455 bio
->bi_size
-= len
<< 9;
457 bch_keylist_add(&s
->insert_keys
,
458 &KEY(op
->inode
, bio
->bi_sector
, len
));
461 op
->insert_data_done
= true;
464 continue_at(cl
, bch_data_insert_keys
, bcache_wq
);
467 static void bch_data_insert_error(struct closure
*cl
)
469 struct btree_op
*op
= container_of(cl
, struct btree_op
, cl
);
470 struct search
*s
= container_of(op
, struct search
, op
);
473 * Our data write just errored, which means we've got a bunch of keys to
474 * insert that point to data that wasn't succesfully written.
476 * We don't have to insert those keys but we still have to invalidate
477 * that region of the cache - so, if we just strip off all the pointers
478 * from the keys we'll accomplish just that.
481 struct bkey
*src
= s
->insert_keys
.keys
, *dst
= s
->insert_keys
.keys
;
483 while (src
!= s
->insert_keys
.top
) {
484 struct bkey
*n
= bkey_next(src
);
486 SET_KEY_PTRS(src
, 0);
487 memmove(dst
, src
, bkey_bytes(src
));
489 dst
= bkey_next(dst
);
493 s
->insert_keys
.top
= dst
;
495 bch_data_insert_keys(cl
);
498 static void bch_data_insert_endio(struct bio
*bio
, int error
)
500 struct closure
*cl
= bio
->bi_private
;
501 struct btree_op
*op
= container_of(cl
, struct btree_op
, cl
);
502 struct search
*s
= container_of(op
, struct search
, op
);
505 /* TODO: We could try to recover from this. */
509 set_closure_fn(cl
, bch_data_insert_error
, bcache_wq
);
511 set_closure_fn(cl
, NULL
, NULL
);
514 bch_bbio_endio(op
->c
, bio
, error
, "writing data to cache");
517 static void bch_data_insert_start(struct closure
*cl
)
519 struct btree_op
*op
= container_of(cl
, struct btree_op
, cl
);
520 struct search
*s
= container_of(op
, struct search
, op
);
521 struct bio
*bio
= op
->cache_bio
, *n
;
524 return bch_data_invalidate(cl
);
526 if (atomic_sub_return(bio_sectors(bio
), &op
->c
->sectors_to_gc
) < 0) {
527 set_gc_sectors(op
->c
);
532 * Journal writes are marked REQ_FLUSH; if the original write was a
533 * flush, it'll wait on the journal write.
535 bio
->bi_rw
&= ~(REQ_FLUSH
|REQ_FUA
);
540 struct bio_set
*split
= s
->d
541 ? s
->d
->bio_split
: op
->c
->bio_split
;
543 /* 1 for the device pointer and 1 for the chksum */
544 if (bch_keylist_realloc(&s
->insert_keys
,
545 1 + (op
->csum
? 1 : 0),
547 continue_at(cl
, bch_data_insert_keys
, bcache_wq
);
549 k
= s
->insert_keys
.top
;
551 SET_KEY_INODE(k
, op
->inode
);
552 SET_KEY_OFFSET(k
, bio
->bi_sector
);
554 if (!bch_alloc_sectors(k
, bio_sectors(bio
), s
))
557 n
= bch_bio_split(bio
, KEY_SIZE(k
), GFP_NOIO
, split
);
559 n
->bi_end_io
= bch_data_insert_endio
;
563 SET_KEY_DIRTY(k
, true);
565 for (i
= 0; i
< KEY_PTRS(k
); i
++)
566 SET_GC_MARK(PTR_BUCKET(op
->c
, k
, i
),
570 SET_KEY_CSUM(k
, op
->csum
);
574 trace_bcache_cache_insert(k
);
575 bch_keylist_push(&s
->insert_keys
);
577 n
->bi_rw
|= REQ_WRITE
;
578 bch_submit_bbio(n
, op
->c
, k
, 0);
581 op
->insert_data_done
= true;
582 continue_at(cl
, bch_data_insert_keys
, bcache_wq
);
584 /* bch_alloc_sectors() blocks if s->writeback = true */
585 BUG_ON(s
->writeback
);
588 * But if it's not a writeback write we'd rather just bail out if
589 * there aren't any buckets ready to write to - it might take awhile and
590 * we might be starving btree writes for gc or something.
595 * Writethrough write: We can't complete the write until we've
596 * updated the index. But we don't want to delay the write while
597 * we wait for buckets to be freed up, so just invalidate the
601 return bch_data_invalidate(cl
);
604 * From a cache miss, we can just insert the keys for the data
605 * we have written or bail out if we didn't do anything.
607 op
->insert_data_done
= true;
610 if (!bch_keylist_empty(&s
->insert_keys
))
611 continue_at(cl
, bch_data_insert_keys
, bcache_wq
);
618 * bch_data_insert - stick some data in the cache
620 * This is the starting point for any data to end up in a cache device; it could
621 * be from a normal write, or a writeback write, or a write to a flash only
622 * volume - it's also used by the moving garbage collector to compact data in
623 * mostly empty buckets.
625 * It first writes the data to the cache, creating a list of keys to be inserted
626 * (if the data had to be fragmented there will be multiple keys); after the
627 * data is written it calls bch_journal, and after the keys have been added to
628 * the next journal write they're inserted into the btree.
630 * It inserts the data in op->cache_bio; bi_sector is used for the key offset,
631 * and op->inode is used for the key inode.
633 * If op->bypass is true, instead of inserting the data it invalidates the
634 * region of the cache represented by op->cache_bio and op->inode.
636 void bch_data_insert(struct closure
*cl
)
638 struct btree_op
*op
= container_of(cl
, struct btree_op
, cl
);
639 struct search
*s
= container_of(op
, struct search
, op
);
641 bch_keylist_init(&s
->insert_keys
);
642 bio_get(op
->cache_bio
);
643 bch_data_insert_start(cl
);
646 /* Common code for the make_request functions */
648 static void request_endio(struct bio
*bio
, int error
)
650 struct closure
*cl
= bio
->bi_private
;
653 struct search
*s
= container_of(cl
, struct search
, cl
);
655 /* Only cache read errors are recoverable */
656 s
->recoverable
= false;
663 void bch_cache_read_endio(struct bio
*bio
, int error
)
665 struct bbio
*b
= container_of(bio
, struct bbio
, bio
);
666 struct closure
*cl
= bio
->bi_private
;
667 struct search
*s
= container_of(cl
, struct search
, cl
);
670 * If the bucket was reused while our bio was in flight, we might have
671 * read the wrong data. Set s->error but not error so it doesn't get
672 * counted against the cache device, but we'll still reread the data
673 * from the backing device.
678 else if (ptr_stale(s
->op
.c
, &b
->key
, 0)) {
679 atomic_long_inc(&s
->op
.c
->cache_read_races
);
683 bch_bbio_endio(s
->op
.c
, bio
, error
, "reading from cache");
686 static void bio_complete(struct search
*s
)
689 int cpu
, rw
= bio_data_dir(s
->orig_bio
);
690 unsigned long duration
= jiffies
- s
->start_time
;
692 cpu
= part_stat_lock();
693 part_round_stats(cpu
, &s
->d
->disk
->part0
);
694 part_stat_add(cpu
, &s
->d
->disk
->part0
, ticks
[rw
], duration
);
697 trace_bcache_request_end(s
, s
->orig_bio
);
698 bio_endio(s
->orig_bio
, s
->error
);
703 static void do_bio_hook(struct search
*s
)
705 struct bio
*bio
= &s
->bio
.bio
;
706 memcpy(bio
, s
->orig_bio
, sizeof(struct bio
));
708 bio
->bi_end_io
= request_endio
;
709 bio
->bi_private
= &s
->cl
;
710 atomic_set(&bio
->bi_cnt
, 3);
713 static void search_free(struct closure
*cl
)
715 struct search
*s
= container_of(cl
, struct search
, cl
);
719 bio_put(s
->op
.cache_bio
);
721 if (s
->unaligned_bvec
)
722 mempool_free(s
->bio
.bio
.bi_io_vec
, s
->d
->unaligned_bvec
);
724 closure_debug_destroy(cl
);
725 mempool_free(s
, s
->d
->c
->search
);
728 static struct search
*search_alloc(struct bio
*bio
, struct bcache_device
*d
)
733 s
= mempool_alloc(d
->c
->search
, GFP_NOIO
);
734 memset(s
, 0, offsetof(struct search
, insert_keys
));
736 __closure_init(&s
->cl
, NULL
);
744 s
->write
= (bio
->bi_rw
& REQ_WRITE
) != 0;
745 s
->op
.flush_journal
= (bio
->bi_rw
& (REQ_FLUSH
|REQ_FUA
)) != 0;
747 s
->start_time
= jiffies
;
750 if (bio
->bi_size
!= bio_segments(bio
) * PAGE_SIZE
) {
751 bv
= mempool_alloc(d
->unaligned_bvec
, GFP_NOIO
);
752 memcpy(bv
, bio_iovec(bio
),
753 sizeof(struct bio_vec
) * bio_segments(bio
));
755 s
->bio
.bio
.bi_io_vec
= bv
;
756 s
->unaligned_bvec
= 1;
762 static void btree_read_async(struct closure
*cl
)
764 struct btree_op
*op
= container_of(cl
, struct btree_op
, cl
);
766 int ret
= btree_root(search_recurse
, op
->c
, op
);
769 continue_at(cl
, btree_read_async
, bcache_wq
);
776 static void cached_dev_bio_complete(struct closure
*cl
)
778 struct search
*s
= container_of(cl
, struct search
, cl
);
779 struct cached_dev
*dc
= container_of(s
->d
, struct cached_dev
, disk
);
785 unsigned bch_get_congested(struct cache_set
*c
)
790 if (!c
->congested_read_threshold_us
&&
791 !c
->congested_write_threshold_us
)
794 i
= (local_clock_us() - c
->congested_last_us
) / 1024;
798 i
+= atomic_read(&c
->congested
);
805 i
= fract_exp_two(i
, 6);
807 rand
= get_random_int();
808 i
-= bitmap_weight(&rand
, BITS_PER_LONG
);
810 return i
> 0 ? i
: 1;
813 static void add_sequential(struct task_struct
*t
)
815 ewma_add(t
->sequential_io_avg
,
816 t
->sequential_io
, 8, 0);
818 t
->sequential_io
= 0;
821 static struct hlist_head
*iohash(struct cached_dev
*dc
, uint64_t k
)
823 return &dc
->io_hash
[hash_64(k
, RECENT_IO_BITS
)];
826 static bool check_should_bypass(struct cached_dev
*dc
, struct search
*s
)
828 struct cache_set
*c
= s
->op
.c
;
829 struct bio
*bio
= &s
->bio
.bio
;
830 unsigned mode
= cache_mode(dc
, bio
);
831 unsigned sectors
, congested
= bch_get_congested(c
);
833 if (atomic_read(&dc
->disk
.detaching
) ||
834 c
->gc_stats
.in_use
> CUTOFF_CACHE_ADD
||
835 (bio
->bi_rw
& REQ_DISCARD
))
838 if (mode
== CACHE_MODE_NONE
||
839 (mode
== CACHE_MODE_WRITEAROUND
&&
840 (bio
->bi_rw
& REQ_WRITE
)))
843 if (bio
->bi_sector
& (c
->sb
.block_size
- 1) ||
844 bio_sectors(bio
) & (c
->sb
.block_size
- 1)) {
845 pr_debug("skipping unaligned io");
849 if (!congested
&& !dc
->sequential_cutoff
)
853 mode
== CACHE_MODE_WRITEBACK
&&
854 (bio
->bi_rw
& REQ_WRITE
) &&
855 (bio
->bi_rw
& REQ_SYNC
))
858 if (dc
->sequential_merge
) {
861 spin_lock(&dc
->io_lock
);
863 hlist_for_each_entry(i
, iohash(dc
, bio
->bi_sector
), hash
)
864 if (i
->last
== bio
->bi_sector
&&
865 time_before(jiffies
, i
->jiffies
))
868 i
= list_first_entry(&dc
->io_lru
, struct io
, lru
);
870 add_sequential(s
->task
);
873 if (i
->sequential
+ bio
->bi_size
> i
->sequential
)
874 i
->sequential
+= bio
->bi_size
;
876 i
->last
= bio_end_sector(bio
);
877 i
->jiffies
= jiffies
+ msecs_to_jiffies(5000);
878 s
->task
->sequential_io
= i
->sequential
;
881 hlist_add_head(&i
->hash
, iohash(dc
, i
->last
));
882 list_move_tail(&i
->lru
, &dc
->io_lru
);
884 spin_unlock(&dc
->io_lock
);
886 s
->task
->sequential_io
= bio
->bi_size
;
888 add_sequential(s
->task
);
891 sectors
= max(s
->task
->sequential_io
,
892 s
->task
->sequential_io_avg
) >> 9;
894 if (dc
->sequential_cutoff
&&
895 sectors
>= dc
->sequential_cutoff
>> 9) {
896 trace_bcache_bypass_sequential(s
->orig_bio
);
900 if (congested
&& sectors
>= congested
) {
901 trace_bcache_bypass_congested(s
->orig_bio
);
906 bch_rescale_priorities(c
, bio_sectors(bio
));
909 bch_mark_sectors_bypassed(s
, bio_sectors(bio
));
915 static void cached_dev_cache_miss_done(struct closure
*cl
)
917 struct search
*s
= container_of(cl
, struct search
, cl
);
919 if (s
->op
.insert_collision
)
920 bch_mark_cache_miss_collision(s
);
922 if (s
->op
.cache_bio
) {
926 __bio_for_each_segment(bv
, s
->op
.cache_bio
, i
, 0)
927 __free_page(bv
->bv_page
);
930 cached_dev_bio_complete(cl
);
933 static void cached_dev_read_error(struct closure
*cl
)
935 struct search
*s
= container_of(cl
, struct search
, cl
);
936 struct bio
*bio
= &s
->bio
.bio
;
940 if (s
->recoverable
) {
941 /* Retry from the backing device: */
942 trace_bcache_read_retry(s
->orig_bio
);
945 bv
= s
->bio
.bio
.bi_io_vec
;
947 s
->bio
.bio
.bi_io_vec
= bv
;
949 if (!s
->unaligned_bvec
)
950 bio_for_each_segment(bv
, s
->orig_bio
, i
)
951 bv
->bv_offset
= 0, bv
->bv_len
= PAGE_SIZE
;
953 memcpy(s
->bio
.bio
.bi_io_vec
,
954 bio_iovec(s
->orig_bio
),
955 sizeof(struct bio_vec
) *
956 bio_segments(s
->orig_bio
));
958 /* XXX: invalidate cache */
960 closure_bio_submit(bio
, cl
, s
->d
);
963 continue_at(cl
, cached_dev_cache_miss_done
, NULL
);
966 static void cached_dev_read_done(struct closure
*cl
)
968 struct search
*s
= container_of(cl
, struct search
, cl
);
969 struct cached_dev
*dc
= container_of(s
->d
, struct cached_dev
, disk
);
972 * We had a cache miss; cache_bio now contains data ready to be inserted
975 * First, we copy the data we just read from cache_bio's bounce buffers
976 * to the buffers the original bio pointed to:
979 if (s
->op
.cache_bio
) {
980 bio_reset(s
->op
.cache_bio
);
981 s
->op
.cache_bio
->bi_sector
= s
->cache_miss
->bi_sector
;
982 s
->op
.cache_bio
->bi_bdev
= s
->cache_miss
->bi_bdev
;
983 s
->op
.cache_bio
->bi_size
= s
->cache_bio_sectors
<< 9;
984 bch_bio_map(s
->op
.cache_bio
, NULL
);
986 bio_copy_data(s
->cache_miss
, s
->op
.cache_bio
);
988 bio_put(s
->cache_miss
);
989 s
->cache_miss
= NULL
;
992 if (verify(dc
, &s
->bio
.bio
) && s
->recoverable
)
997 if (s
->op
.cache_bio
&&
998 !test_bit(CACHE_SET_STOPPING
, &s
->op
.c
->flags
)) {
999 s
->op
.type
= BTREE_REPLACE
;
1000 closure_call(&s
->op
.cl
, bch_data_insert
, NULL
, cl
);
1003 continue_at(cl
, cached_dev_cache_miss_done
, NULL
);
1006 static void cached_dev_read_done_bh(struct closure
*cl
)
1008 struct search
*s
= container_of(cl
, struct search
, cl
);
1009 struct cached_dev
*dc
= container_of(s
->d
, struct cached_dev
, disk
);
1011 bch_mark_cache_accounting(s
, !s
->cache_miss
, s
->op
.bypass
);
1012 trace_bcache_read(s
->orig_bio
, !s
->cache_miss
, s
->op
.bypass
);
1015 continue_at_nobarrier(cl
, cached_dev_read_error
, bcache_wq
);
1016 else if (s
->op
.cache_bio
|| verify(dc
, &s
->bio
.bio
))
1017 continue_at_nobarrier(cl
, cached_dev_read_done
, bcache_wq
);
1019 continue_at_nobarrier(cl
, cached_dev_bio_complete
, NULL
);
1022 static int cached_dev_cache_miss(struct btree
*b
, struct search
*s
,
1023 struct bio
*bio
, unsigned sectors
)
1027 struct cached_dev
*dc
= container_of(s
->d
, struct cached_dev
, disk
);
1028 struct bio
*miss
, *cache_bio
;
1030 if (s
->cache_miss
|| s
->op
.bypass
) {
1031 miss
= bch_bio_split(bio
, sectors
, GFP_NOIO
, s
->d
->bio_split
);
1033 s
->op
.lookup_done
= true;
1037 if (!(bio
->bi_rw
& REQ_RAHEAD
) &&
1038 !(bio
->bi_rw
& REQ_META
) &&
1039 s
->op
.c
->gc_stats
.in_use
< CUTOFF_CACHE_READA
)
1040 reada
= min_t(sector_t
, dc
->readahead
>> 9,
1041 bdev_sectors(bio
->bi_bdev
) - bio_end_sector(bio
));
1043 s
->cache_bio_sectors
= min(sectors
, bio_sectors(bio
) + reada
);
1045 s
->op
.replace
= KEY(s
->op
.inode
, bio
->bi_sector
+
1046 s
->cache_bio_sectors
, s
->cache_bio_sectors
);
1048 ret
= bch_btree_insert_check_key(b
, &s
->op
, &s
->op
.replace
);
1052 miss
= bch_bio_split(bio
, sectors
, GFP_NOIO
, s
->d
->bio_split
);
1054 s
->op
.lookup_done
= true;
1056 /* btree_search_recurse()'s btree iterator is no good anymore */
1059 cache_bio
= bio_alloc_bioset(GFP_NOWAIT
,
1060 DIV_ROUND_UP(s
->cache_bio_sectors
, PAGE_SECTORS
),
1061 dc
->disk
.bio_split
);
1065 cache_bio
->bi_sector
= miss
->bi_sector
;
1066 cache_bio
->bi_bdev
= miss
->bi_bdev
;
1067 cache_bio
->bi_size
= s
->cache_bio_sectors
<< 9;
1069 cache_bio
->bi_end_io
= request_endio
;
1070 cache_bio
->bi_private
= &s
->cl
;
1072 bch_bio_map(cache_bio
, NULL
);
1073 if (bio_alloc_pages(cache_bio
, __GFP_NOWARN
|GFP_NOIO
))
1076 s
->cache_miss
= miss
;
1077 s
->op
.cache_bio
= cache_bio
;
1079 closure_bio_submit(cache_bio
, &s
->cl
, s
->d
);
1085 miss
->bi_end_io
= request_endio
;
1086 miss
->bi_private
= &s
->cl
;
1087 closure_bio_submit(miss
, &s
->cl
, s
->d
);
1091 static void cached_dev_read(struct cached_dev
*dc
, struct search
*s
)
1093 struct closure
*cl
= &s
->cl
;
1095 closure_call(&s
->op
.cl
, btree_read_async
, NULL
, cl
);
1096 continue_at(cl
, cached_dev_read_done_bh
, NULL
);
1099 /* Process writes */
1101 static void cached_dev_write_complete(struct closure
*cl
)
1103 struct search
*s
= container_of(cl
, struct search
, cl
);
1104 struct cached_dev
*dc
= container_of(s
->d
, struct cached_dev
, disk
);
1106 up_read_non_owner(&dc
->writeback_lock
);
1107 cached_dev_bio_complete(cl
);
1110 static void cached_dev_write(struct cached_dev
*dc
, struct search
*s
)
1112 struct closure
*cl
= &s
->cl
;
1113 struct bio
*bio
= &s
->bio
.bio
;
1114 struct bkey start
= KEY(dc
->disk
.id
, bio
->bi_sector
, 0);
1115 struct bkey end
= KEY(dc
->disk
.id
, bio_end_sector(bio
), 0);
1117 bch_keybuf_check_overlapping(&s
->op
.c
->moving_gc_keys
, &start
, &end
);
1119 down_read_non_owner(&dc
->writeback_lock
);
1120 if (bch_keybuf_check_overlapping(&dc
->writeback_keys
, &start
, &end
)) {
1122 * We overlap with some dirty data undergoing background
1123 * writeback, force this write to writeback
1125 s
->op
.bypass
= false;
1126 s
->writeback
= true;
1130 * Discards aren't _required_ to do anything, so skipping if
1131 * check_overlapping returned true is ok
1133 * But check_overlapping drops dirty keys for which io hasn't started,
1134 * so we still want to call it.
1136 if (bio
->bi_rw
& REQ_DISCARD
)
1137 s
->op
.bypass
= true;
1139 if (should_writeback(dc
, s
->orig_bio
,
1140 cache_mode(dc
, bio
),
1142 s
->op
.bypass
= false;
1143 s
->writeback
= true;
1146 trace_bcache_write(s
->orig_bio
, s
->writeback
, s
->op
.bypass
);
1149 s
->op
.cache_bio
= s
->orig_bio
;
1150 bio_get(s
->op
.cache_bio
);
1152 if (!(bio
->bi_rw
& REQ_DISCARD
) ||
1153 blk_queue_discard(bdev_get_queue(dc
->bdev
)))
1154 closure_bio_submit(bio
, cl
, s
->d
);
1155 } else if (s
->writeback
) {
1156 bch_writeback_add(dc
);
1157 s
->op
.cache_bio
= bio
;
1159 if (bio
->bi_rw
& REQ_FLUSH
) {
1160 /* Also need to send a flush to the backing device */
1161 struct bio
*flush
= bio_alloc_bioset(GFP_NOIO
, 0,
1162 dc
->disk
.bio_split
);
1164 flush
->bi_rw
= WRITE_FLUSH
;
1165 flush
->bi_bdev
= bio
->bi_bdev
;
1166 flush
->bi_end_io
= request_endio
;
1167 flush
->bi_private
= cl
;
1169 closure_bio_submit(flush
, cl
, s
->d
);
1172 s
->op
.cache_bio
= bio_clone_bioset(bio
, GFP_NOIO
,
1173 dc
->disk
.bio_split
);
1175 closure_bio_submit(bio
, cl
, s
->d
);
1178 closure_call(&s
->op
.cl
, bch_data_insert
, NULL
, cl
);
1179 continue_at(cl
, cached_dev_write_complete
, NULL
);
1182 static void cached_dev_nodata(struct closure
*cl
)
1184 struct search
*s
= container_of(cl
, struct search
, cl
);
1185 struct bio
*bio
= &s
->bio
.bio
;
1187 if (s
->op
.flush_journal
)
1188 bch_journal_meta(s
->op
.c
, cl
);
1190 /* If it's a flush, we send the flush to the backing device too */
1191 closure_bio_submit(bio
, cl
, s
->d
);
1193 continue_at(cl
, cached_dev_bio_complete
, NULL
);
1196 /* Cached devices - read & write stuff */
1198 static void cached_dev_make_request(struct request_queue
*q
, struct bio
*bio
)
1201 struct bcache_device
*d
= bio
->bi_bdev
->bd_disk
->private_data
;
1202 struct cached_dev
*dc
= container_of(d
, struct cached_dev
, disk
);
1203 int cpu
, rw
= bio_data_dir(bio
);
1205 cpu
= part_stat_lock();
1206 part_stat_inc(cpu
, &d
->disk
->part0
, ios
[rw
]);
1207 part_stat_add(cpu
, &d
->disk
->part0
, sectors
[rw
], bio_sectors(bio
));
1210 bio
->bi_bdev
= dc
->bdev
;
1211 bio
->bi_sector
+= dc
->sb
.data_offset
;
1213 if (cached_dev_get(dc
)) {
1214 s
= search_alloc(bio
, d
);
1215 trace_bcache_request_start(s
, bio
);
1217 if (!bio
->bi_size
) {
1219 * can't call bch_journal_meta from under
1220 * generic_make_request
1222 continue_at_nobarrier(&s
->cl
,
1226 s
->op
.bypass
= check_should_bypass(dc
, s
);
1229 cached_dev_write(dc
, s
);
1231 cached_dev_read(dc
, s
);
1234 if ((bio
->bi_rw
& REQ_DISCARD
) &&
1235 !blk_queue_discard(bdev_get_queue(dc
->bdev
)))
1238 bch_generic_make_request(bio
, &d
->bio_split_hook
);
1242 static int cached_dev_ioctl(struct bcache_device
*d
, fmode_t mode
,
1243 unsigned int cmd
, unsigned long arg
)
1245 struct cached_dev
*dc
= container_of(d
, struct cached_dev
, disk
);
1246 return __blkdev_driver_ioctl(dc
->bdev
, mode
, cmd
, arg
);
1249 static int cached_dev_congested(void *data
, int bits
)
1251 struct bcache_device
*d
= data
;
1252 struct cached_dev
*dc
= container_of(d
, struct cached_dev
, disk
);
1253 struct request_queue
*q
= bdev_get_queue(dc
->bdev
);
1256 if (bdi_congested(&q
->backing_dev_info
, bits
))
1259 if (cached_dev_get(dc
)) {
1263 for_each_cache(ca
, d
->c
, i
) {
1264 q
= bdev_get_queue(ca
->bdev
);
1265 ret
|= bdi_congested(&q
->backing_dev_info
, bits
);
1274 void bch_cached_dev_request_init(struct cached_dev
*dc
)
1276 struct gendisk
*g
= dc
->disk
.disk
;
1278 g
->queue
->make_request_fn
= cached_dev_make_request
;
1279 g
->queue
->backing_dev_info
.congested_fn
= cached_dev_congested
;
1280 dc
->disk
.cache_miss
= cached_dev_cache_miss
;
1281 dc
->disk
.ioctl
= cached_dev_ioctl
;
1284 /* Flash backed devices */
1286 static int flash_dev_cache_miss(struct btree
*b
, struct search
*s
,
1287 struct bio
*bio
, unsigned sectors
)
1294 bio_for_each_segment(bv
, bio
, i
) {
1295 unsigned j
= min(bv
->bv_len
>> 9, sectors
);
1297 void *p
= kmap(bv
->bv_page
);
1298 memset(p
+ bv
->bv_offset
, 0, j
<< 9);
1299 kunmap(bv
->bv_page
);
1304 bio_advance(bio
, min(sectors
<< 9, bio
->bi_size
));
1307 s
->op
.lookup_done
= true;
1312 static void flash_dev_nodata(struct closure
*cl
)
1314 struct search
*s
= container_of(cl
, struct search
, cl
);
1316 if (s
->op
.flush_journal
)
1317 bch_journal_meta(s
->op
.c
, cl
);
1319 continue_at(cl
, search_free
, NULL
);
1322 static void flash_dev_make_request(struct request_queue
*q
, struct bio
*bio
)
1326 struct bcache_device
*d
= bio
->bi_bdev
->bd_disk
->private_data
;
1327 int cpu
, rw
= bio_data_dir(bio
);
1329 cpu
= part_stat_lock();
1330 part_stat_inc(cpu
, &d
->disk
->part0
, ios
[rw
]);
1331 part_stat_add(cpu
, &d
->disk
->part0
, sectors
[rw
], bio_sectors(bio
));
1334 s
= search_alloc(bio
, d
);
1338 trace_bcache_request_start(s
, bio
);
1340 if (!bio
->bi_size
) {
1342 * can't call bch_journal_meta from under
1343 * generic_make_request
1345 continue_at_nobarrier(&s
->cl
,
1349 bch_keybuf_check_overlapping(&s
->op
.c
->moving_gc_keys
,
1350 &KEY(d
->id
, bio
->bi_sector
, 0),
1351 &KEY(d
->id
, bio_end_sector(bio
), 0));
1353 s
->op
.bypass
= (bio
->bi_rw
& REQ_DISCARD
) != 0;
1354 s
->writeback
= true;
1355 s
->op
.cache_bio
= bio
;
1357 closure_call(&s
->op
.cl
, bch_data_insert
, NULL
, cl
);
1359 closure_call(&s
->op
.cl
, btree_read_async
, NULL
, cl
);
1362 continue_at(cl
, search_free
, NULL
);
1365 static int flash_dev_ioctl(struct bcache_device
*d
, fmode_t mode
,
1366 unsigned int cmd
, unsigned long arg
)
1371 static int flash_dev_congested(void *data
, int bits
)
1373 struct bcache_device
*d
= data
;
1374 struct request_queue
*q
;
1379 for_each_cache(ca
, d
->c
, i
) {
1380 q
= bdev_get_queue(ca
->bdev
);
1381 ret
|= bdi_congested(&q
->backing_dev_info
, bits
);
1387 void bch_flash_dev_request_init(struct bcache_device
*d
)
1389 struct gendisk
*g
= d
->disk
;
1391 g
->queue
->make_request_fn
= flash_dev_make_request
;
1392 g
->queue
->backing_dev_info
.congested_fn
= flash_dev_congested
;
1393 d
->cache_miss
= flash_dev_cache_miss
;
1394 d
->ioctl
= flash_dev_ioctl
;
1397 void bch_request_exit(void)
1399 #ifdef CONFIG_CGROUP_BCACHE
1400 cgroup_unload_subsys(&bcache_subsys
);
1402 if (bch_search_cache
)
1403 kmem_cache_destroy(bch_search_cache
);
1406 int __init
bch_request_init(void)
1408 bch_search_cache
= KMEM_CACHE(search
, 0);
1409 if (!bch_search_cache
)
1412 #ifdef CONFIG_CGROUP_BCACHE
1413 cgroup_load_subsys(&bcache_subsys
);
1414 init_bch_cgroup(&bcache_default_cgroup
);
1416 cgroup_add_cftypes(&bcache_subsys
, bch_files
);