2 * background writeback - scan btree for dirty data and write it to the backing
5 * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
6 * Copyright 2012 Google, Inc.
12 #include "writeback.h"
14 #include <linux/delay.h>
15 #include <linux/freezer.h>
16 #include <linux/kthread.h>
17 #include <trace/events/bcache.h>
21 static void __update_writeback_rate(struct cached_dev
*dc
)
23 struct cache_set
*c
= dc
->disk
.c
;
24 uint64_t cache_sectors
= c
->nbuckets
* c
->sb
.bucket_size
;
25 uint64_t cache_dirty_target
=
26 div_u64(cache_sectors
* dc
->writeback_percent
, 100);
28 int64_t target
= div64_u64(cache_dirty_target
* bdev_sectors(dc
->bdev
),
29 c
->cached_dev_sectors
);
35 int64_t dirty
= bcache_dev_sectors_dirty(&dc
->disk
);
36 int64_t derivative
= dirty
- dc
->disk
.sectors_dirty_last
;
38 dc
->disk
.sectors_dirty_last
= dirty
;
40 derivative
*= dc
->writeback_rate_d_term
;
41 derivative
= clamp(derivative
, -dirty
, dirty
);
43 derivative
= ewma_add(dc
->disk
.sectors_dirty_derivative
, derivative
,
44 dc
->writeback_rate_d_smooth
, 0);
46 /* Avoid divide by zero */
50 error
= div64_s64((dirty
+ derivative
- target
) << 8, target
);
52 change
= div_s64((dc
->writeback_rate
.rate
* error
) >> 8,
53 dc
->writeback_rate_p_term_inverse
);
55 /* Don't increase writeback rate if the device isn't keeping up */
57 time_after64(local_clock(),
58 dc
->writeback_rate
.next
+ 10 * NSEC_PER_MSEC
))
61 dc
->writeback_rate
.rate
=
62 clamp_t(int64_t, dc
->writeback_rate
.rate
+ change
,
65 dc
->writeback_rate_derivative
= derivative
;
66 dc
->writeback_rate_change
= change
;
67 dc
->writeback_rate_target
= target
;
70 static void update_writeback_rate(struct work_struct
*work
)
72 struct cached_dev
*dc
= container_of(to_delayed_work(work
),
74 writeback_rate_update
);
76 down_read(&dc
->writeback_lock
);
78 if (atomic_read(&dc
->has_dirty
) &&
79 dc
->writeback_percent
)
80 __update_writeback_rate(dc
);
82 up_read(&dc
->writeback_lock
);
84 schedule_delayed_work(&dc
->writeback_rate_update
,
85 dc
->writeback_rate_update_seconds
* HZ
);
88 static unsigned writeback_delay(struct cached_dev
*dc
, unsigned sectors
)
92 if (test_bit(BCACHE_DEV_DETACHING
, &dc
->disk
.flags
) ||
93 !dc
->writeback_percent
)
96 ret
= bch_next_delay(&dc
->writeback_rate
, sectors
* 10000000ULL);
98 return min_t(uint64_t, ret
, HZ
);
103 struct cached_dev
*dc
;
107 static void dirty_init(struct keybuf_key
*w
)
109 struct dirty_io
*io
= w
->private;
110 struct bio
*bio
= &io
->bio
;
113 if (!io
->dc
->writeback_percent
)
114 bio_set_prio(bio
, IOPRIO_PRIO_VALUE(IOPRIO_CLASS_IDLE
, 0));
116 bio
->bi_size
= KEY_SIZE(&w
->key
) << 9;
117 bio
->bi_max_vecs
= DIV_ROUND_UP(KEY_SIZE(&w
->key
), PAGE_SECTORS
);
119 bio
->bi_io_vec
= bio
->bi_inline_vecs
;
120 bch_bio_map(bio
, NULL
);
123 static void dirty_io_destructor(struct closure
*cl
)
125 struct dirty_io
*io
= container_of(cl
, struct dirty_io
, cl
);
129 static void write_dirty_finish(struct closure
*cl
)
131 struct dirty_io
*io
= container_of(cl
, struct dirty_io
, cl
);
132 struct keybuf_key
*w
= io
->bio
.bi_private
;
133 struct cached_dev
*dc
= io
->dc
;
137 bio_for_each_segment_all(bv
, &io
->bio
, i
)
138 __free_page(bv
->bv_page
);
140 /* This is kind of a dumb way of signalling errors. */
141 if (KEY_DIRTY(&w
->key
)) {
146 bch_keylist_init(&keys
);
148 bkey_copy(keys
.top
, &w
->key
);
149 SET_KEY_DIRTY(keys
.top
, false);
150 bch_keylist_push(&keys
);
152 for (i
= 0; i
< KEY_PTRS(&w
->key
); i
++)
153 atomic_inc(&PTR_BUCKET(dc
->disk
.c
, &w
->key
, i
)->pin
);
155 ret
= bch_btree_insert(dc
->disk
.c
, &keys
, NULL
, &w
->key
);
158 trace_bcache_writeback_collision(&w
->key
);
161 ? &dc
->disk
.c
->writeback_keys_failed
162 : &dc
->disk
.c
->writeback_keys_done
);
165 bch_keybuf_del(&dc
->writeback_keys
, w
);
168 closure_return_with_destructor(cl
, dirty_io_destructor
);
171 static void dirty_endio(struct bio
*bio
, int error
)
173 struct keybuf_key
*w
= bio
->bi_private
;
174 struct dirty_io
*io
= w
->private;
177 SET_KEY_DIRTY(&w
->key
, false);
179 closure_put(&io
->cl
);
182 static void write_dirty(struct closure
*cl
)
184 struct dirty_io
*io
= container_of(cl
, struct dirty_io
, cl
);
185 struct keybuf_key
*w
= io
->bio
.bi_private
;
188 io
->bio
.bi_rw
= WRITE
;
189 io
->bio
.bi_sector
= KEY_START(&w
->key
);
190 io
->bio
.bi_bdev
= io
->dc
->bdev
;
191 io
->bio
.bi_end_io
= dirty_endio
;
193 closure_bio_submit(&io
->bio
, cl
, &io
->dc
->disk
);
195 continue_at(cl
, write_dirty_finish
, system_wq
);
198 static void read_dirty_endio(struct bio
*bio
, int error
)
200 struct keybuf_key
*w
= bio
->bi_private
;
201 struct dirty_io
*io
= w
->private;
203 bch_count_io_errors(PTR_CACHE(io
->dc
->disk
.c
, &w
->key
, 0),
204 error
, "reading dirty data from cache");
206 dirty_endio(bio
, error
);
209 static void read_dirty_submit(struct closure
*cl
)
211 struct dirty_io
*io
= container_of(cl
, struct dirty_io
, cl
);
213 closure_bio_submit(&io
->bio
, cl
, &io
->dc
->disk
);
215 continue_at(cl
, write_dirty
, system_wq
);
218 static void read_dirty(struct cached_dev
*dc
)
221 struct keybuf_key
*w
;
225 closure_init_stack(&cl
);
228 * XXX: if we error, background writeback just spins. Should use some
232 while (!kthread_should_stop()) {
235 w
= bch_keybuf_next(&dc
->writeback_keys
);
239 BUG_ON(ptr_stale(dc
->disk
.c
, &w
->key
, 0));
241 if (KEY_START(&w
->key
) != dc
->last_read
||
242 jiffies_to_msecs(delay
) > 50)
243 while (!kthread_should_stop() && delay
)
244 delay
= schedule_timeout_interruptible(delay
);
246 dc
->last_read
= KEY_OFFSET(&w
->key
);
248 io
= kzalloc(sizeof(struct dirty_io
) + sizeof(struct bio_vec
)
249 * DIV_ROUND_UP(KEY_SIZE(&w
->key
), PAGE_SECTORS
),
258 io
->bio
.bi_sector
= PTR_OFFSET(&w
->key
, 0);
259 io
->bio
.bi_bdev
= PTR_CACHE(dc
->disk
.c
,
261 io
->bio
.bi_rw
= READ
;
262 io
->bio
.bi_end_io
= read_dirty_endio
;
264 if (bio_alloc_pages(&io
->bio
, GFP_KERNEL
))
267 trace_bcache_writeback(&w
->key
);
269 down(&dc
->in_flight
);
270 closure_call(&io
->cl
, read_dirty_submit
, NULL
, &cl
);
272 delay
= writeback_delay(dc
, KEY_SIZE(&w
->key
));
279 bch_keybuf_del(&dc
->writeback_keys
, w
);
283 * Wait for outstanding writeback IOs to finish (and keybuf slots to be
284 * freed) before refilling again
289 /* Scan for dirty data */
291 void bcache_dev_sectors_dirty_add(struct cache_set
*c
, unsigned inode
,
292 uint64_t offset
, int nr_sectors
)
294 struct bcache_device
*d
= c
->devices
[inode
];
295 unsigned stripe_offset
, stripe
, sectors_dirty
;
300 stripe
= offset_to_stripe(d
, offset
);
301 stripe_offset
= offset
& (d
->stripe_size
- 1);
304 int s
= min_t(unsigned, abs(nr_sectors
),
305 d
->stripe_size
- stripe_offset
);
310 if (stripe
>= d
->nr_stripes
)
313 sectors_dirty
= atomic_add_return(s
,
314 d
->stripe_sectors_dirty
+ stripe
);
315 if (sectors_dirty
== d
->stripe_size
)
316 set_bit(stripe
, d
->full_dirty_stripes
);
318 clear_bit(stripe
, d
->full_dirty_stripes
);
326 static bool dirty_pred(struct keybuf
*buf
, struct bkey
*k
)
331 static void refill_full_stripes(struct cached_dev
*dc
)
333 struct keybuf
*buf
= &dc
->writeback_keys
;
334 unsigned start_stripe
, stripe
, next_stripe
;
335 bool wrapped
= false;
337 stripe
= offset_to_stripe(&dc
->disk
, KEY_OFFSET(&buf
->last_scanned
));
339 if (stripe
>= dc
->disk
.nr_stripes
)
342 start_stripe
= stripe
;
345 stripe
= find_next_bit(dc
->disk
.full_dirty_stripes
,
346 dc
->disk
.nr_stripes
, stripe
);
348 if (stripe
== dc
->disk
.nr_stripes
)
351 next_stripe
= find_next_zero_bit(dc
->disk
.full_dirty_stripes
,
352 dc
->disk
.nr_stripes
, stripe
);
354 buf
->last_scanned
= KEY(dc
->disk
.id
,
355 stripe
* dc
->disk
.stripe_size
, 0);
357 bch_refill_keybuf(dc
->disk
.c
, buf
,
359 next_stripe
* dc
->disk
.stripe_size
, 0),
362 if (array_freelist_empty(&buf
->freelist
))
365 stripe
= next_stripe
;
367 if (wrapped
&& stripe
> start_stripe
)
370 if (stripe
== dc
->disk
.nr_stripes
) {
377 static bool refill_dirty(struct cached_dev
*dc
)
379 struct keybuf
*buf
= &dc
->writeback_keys
;
380 struct bkey end
= KEY(dc
->disk
.id
, MAX_KEY_OFFSET
, 0);
381 bool searched_from_start
= false;
383 if (dc
->partial_stripes_expensive
) {
384 refill_full_stripes(dc
);
385 if (array_freelist_empty(&buf
->freelist
))
389 if (bkey_cmp(&buf
->last_scanned
, &end
) >= 0) {
390 buf
->last_scanned
= KEY(dc
->disk
.id
, 0, 0);
391 searched_from_start
= true;
394 bch_refill_keybuf(dc
->disk
.c
, buf
, &end
, dirty_pred
);
396 return bkey_cmp(&buf
->last_scanned
, &end
) >= 0 && searched_from_start
;
399 static int bch_writeback_thread(void *arg
)
401 struct cached_dev
*dc
= arg
;
402 bool searched_full_index
;
404 while (!kthread_should_stop()) {
405 down_write(&dc
->writeback_lock
);
406 if (!atomic_read(&dc
->has_dirty
) ||
407 (!test_bit(BCACHE_DEV_DETACHING
, &dc
->disk
.flags
) &&
408 !dc
->writeback_running
)) {
409 up_write(&dc
->writeback_lock
);
410 set_current_state(TASK_INTERRUPTIBLE
);
412 if (kthread_should_stop())
420 searched_full_index
= refill_dirty(dc
);
422 if (searched_full_index
&&
423 RB_EMPTY_ROOT(&dc
->writeback_keys
.keys
)) {
424 atomic_set(&dc
->has_dirty
, 0);
426 SET_BDEV_STATE(&dc
->sb
, BDEV_STATE_CLEAN
);
427 bch_write_bdev_super(dc
, NULL
);
430 up_write(&dc
->writeback_lock
);
432 bch_ratelimit_reset(&dc
->writeback_rate
);
435 if (searched_full_index
) {
436 unsigned delay
= dc
->writeback_delay
* HZ
;
439 !kthread_should_stop() &&
440 !test_bit(BCACHE_DEV_DETACHING
, &dc
->disk
.flags
))
441 delay
= schedule_timeout_interruptible(delay
);
450 struct sectors_dirty_init
{
455 static int sectors_dirty_init_fn(struct btree_op
*_op
, struct btree
*b
,
458 struct sectors_dirty_init
*op
= container_of(_op
,
459 struct sectors_dirty_init
, op
);
460 if (KEY_INODE(k
) > op
->inode
)
464 bcache_dev_sectors_dirty_add(b
->c
, KEY_INODE(k
),
465 KEY_START(k
), KEY_SIZE(k
));
470 void bch_sectors_dirty_init(struct cached_dev
*dc
)
472 struct sectors_dirty_init op
;
474 bch_btree_op_init(&op
.op
, -1);
475 op
.inode
= dc
->disk
.id
;
477 bch_btree_map_keys(&op
.op
, dc
->disk
.c
, &KEY(op
.inode
, 0, 0),
478 sectors_dirty_init_fn
, 0);
481 int bch_cached_dev_writeback_init(struct cached_dev
*dc
)
483 sema_init(&dc
->in_flight
, 64);
484 init_rwsem(&dc
->writeback_lock
);
485 bch_keybuf_init(&dc
->writeback_keys
);
487 dc
->writeback_metadata
= true;
488 dc
->writeback_running
= true;
489 dc
->writeback_percent
= 10;
490 dc
->writeback_delay
= 30;
491 dc
->writeback_rate
.rate
= 1024;
493 dc
->writeback_rate_update_seconds
= 30;
494 dc
->writeback_rate_d_term
= 16;
495 dc
->writeback_rate_p_term_inverse
= 64;
496 dc
->writeback_rate_d_smooth
= 8;
498 dc
->writeback_thread
= kthread_create(bch_writeback_thread
, dc
,
500 if (IS_ERR(dc
->writeback_thread
))
501 return PTR_ERR(dc
->writeback_thread
);
503 set_task_state(dc
->writeback_thread
, TASK_INTERRUPTIBLE
);
505 INIT_DELAYED_WORK(&dc
->writeback_rate_update
, update_writeback_rate
);
506 schedule_delayed_work(&dc
->writeback_rate_update
,
507 dc
->writeback_rate_update_seconds
* HZ
);