projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
btrfs: Separate scrub_blocked_if_needed() to scrub_pause_on/off()
[deliverable/linux.git] / fs / btrfs / scrub.c
1 /*
2 * Copyright (C) 2011, 2012 STRATO. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
17 */
18
19 #include <linux/blkdev.h>
20 #include <linux/ratelimit.h>
21 #include "ctree.h"
22 #include "volumes.h"
23 #include "disk-io.h"
24 #include "ordered-data.h"
25 #include "transaction.h"
26 #include "backref.h"
27 #include "extent_io.h"
28 #include "dev-replace.h"
29 #include "check-integrity.h"
30 #include "rcu-string.h"
31 #include "raid56.h"
32
33 /*
34 * This is only the first step towards a full-features scrub. It reads all
35 * extent and super block and verifies the checksums. In case a bad checksum
36 * is found or the extent cannot be read, good data will be written back if
37 * any can be found.
38 *
39 * Future enhancements:
40 * - In case an unrepairable extent is encountered, track which files are
41 * affected and report them
42 * - track and record media errors, throw out bad devices
43 * - add a mode to also read unallocated space
44 */
45
46 struct scrub_block;
47 struct scrub_ctx;
48
49 /*
50 * the following three values only influence the performance.
51 * The last one configures the number of parallel and outstanding I/O
52 * operations. The first two values configure an upper limit for the number
53 * of (dynamically allocated) pages that are added to a bio.
54 */
55 #define SCRUB_PAGES_PER_RD_BIO 32 /* 128k per bio */
56 #define SCRUB_PAGES_PER_WR_BIO 32 /* 128k per bio */
57 #define SCRUB_BIOS_PER_SCTX 64 /* 8MB per device in flight */
58
59 /*
60 * the following value times PAGE_SIZE needs to be large enough to match the
61 * largest node/leaf/sector size that shall be supported.
62 * Values larger than BTRFS_STRIPE_LEN are not supported.
63 */
64 #define SCRUB_MAX_PAGES_PER_BLOCK 16 /* 64k per node/leaf/sector */
65
66 struct scrub_recover {
67 atomic_t refs;
68 struct btrfs_bio *bbio;
69 u64 map_length;
70 };
71
72 struct scrub_page {
73 struct scrub_block *sblock;
74 struct page *page;
75 struct btrfs_device *dev;
76 struct list_head list;
77 u64 flags; /* extent flags */
78 u64 generation;
79 u64 logical;
80 u64 physical;
81 u64 physical_for_dev_replace;
82 atomic_t refs;
83 struct {
84 unsigned int mirror_num:8;
85 unsigned int have_csum:1;
86 unsigned int io_error:1;
87 };
88 u8 csum[BTRFS_CSUM_SIZE];
89
90 struct scrub_recover *recover;
91 };
92
93 struct scrub_bio {
94 int index;
95 struct scrub_ctx *sctx;
96 struct btrfs_device *dev;
97 struct bio *bio;
98 int err;
99 u64 logical;
100 u64 physical;
101 #if SCRUB_PAGES_PER_WR_BIO >= SCRUB_PAGES_PER_RD_BIO
102 struct scrub_page *pagev[SCRUB_PAGES_PER_WR_BIO];
103 #else
104 struct scrub_page *pagev[SCRUB_PAGES_PER_RD_BIO];
105 #endif
106 int page_count;
107 int next_free;
108 struct btrfs_work work;
109 };
110
111 struct scrub_block {
112 struct scrub_page *pagev[SCRUB_MAX_PAGES_PER_BLOCK];
113 int page_count;
114 atomic_t outstanding_pages;
115 atomic_t refs; /* free mem on transition to zero */
116 struct scrub_ctx *sctx;
117 struct scrub_parity *sparity;
118 struct {
119 unsigned int header_error:1;
120 unsigned int checksum_error:1;
121 unsigned int no_io_error_seen:1;
122 unsigned int generation_error:1; /* also sets header_error */
123
124 /* The following is for the data used to check parity */
125 /* It is for the data with checksum */
126 unsigned int data_corrected:1;
127 };
128 };
129
130 /* Used for the chunks with parity stripe such RAID5/6 */
131 struct scrub_parity {
132 struct scrub_ctx *sctx;
133
134 struct btrfs_device *scrub_dev;
135
136 u64 logic_start;
137
138 u64 logic_end;
139
140 int nsectors;
141
142 int stripe_len;
143
144 atomic_t refs;
145
146 struct list_head spages;
147
148 /* Work of parity check and repair */
149 struct btrfs_work work;
150
151 /* Mark the parity blocks which have data */
152 unsigned long *dbitmap;
153
154 /*
155 * Mark the parity blocks which have data, but errors happen when
156 * read data or check data
157 */
158 unsigned long *ebitmap;
159
160 unsigned long bitmap[0];
161 };
162
163 struct scrub_wr_ctx {
164 struct scrub_bio *wr_curr_bio;
165 struct btrfs_device *tgtdev;
166 int pages_per_wr_bio; /* <= SCRUB_PAGES_PER_WR_BIO */
167 atomic_t flush_all_writes;
168 struct mutex wr_lock;
169 };
170
171 struct scrub_ctx {
172 struct scrub_bio *bios[SCRUB_BIOS_PER_SCTX];
173 struct btrfs_root *dev_root;
174 int first_free;
175 int curr;
176 atomic_t bios_in_flight;
177 atomic_t workers_pending;
178 spinlock_t list_lock;
179 wait_queue_head_t list_wait;
180 u16 csum_size;
181 struct list_head csum_list;
182 atomic_t cancel_req;
183 int readonly;
184 int pages_per_rd_bio;
185 u32 sectorsize;
186 u32 nodesize;
187
188 int is_dev_replace;
189 struct scrub_wr_ctx wr_ctx;
190
191 /*
192 * statistics
193 */
194 struct btrfs_scrub_progress stat;
195 spinlock_t stat_lock;
196
197 /*
198 * Use a ref counter to avoid use-after-free issues. Scrub workers
199 * decrement bios_in_flight and workers_pending and then do a wakeup
200 * on the list_wait wait queue. We must ensure the main scrub task
201 * doesn't free the scrub context before or while the workers are
202 * doing the wakeup() call.
203 */
204 atomic_t refs;
205 };
206
207 struct scrub_fixup_nodatasum {
208 struct scrub_ctx *sctx;
209 struct btrfs_device *dev;
210 u64 logical;
211 struct btrfs_root *root;
212 struct btrfs_work work;
213 int mirror_num;
214 };
215
216 struct scrub_nocow_inode {
217 u64 inum;
218 u64 offset;
219 u64 root;
220 struct list_head list;
221 };
222
223 struct scrub_copy_nocow_ctx {
224 struct scrub_ctx *sctx;
225 u64 logical;
226 u64 len;
227 int mirror_num;
228 u64 physical_for_dev_replace;
229 struct list_head inodes;
230 struct btrfs_work work;
231 };
232
233 struct scrub_warning {
234 struct btrfs_path *path;
235 u64 extent_item_size;
236 const char *errstr;
237 sector_t sector;
238 u64 logical;
239 struct btrfs_device *dev;
240 };
241
242 static void scrub_pending_bio_inc(struct scrub_ctx *sctx);
243 static void scrub_pending_bio_dec(struct scrub_ctx *sctx);
244 static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx);
245 static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx);
246 static int scrub_handle_errored_block(struct scrub_block *sblock_to_check);
247 static int scrub_setup_recheck_block(struct scrub_block *original_sblock,
248 struct scrub_block *sblocks_for_recheck);
249 static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
250 struct scrub_block *sblock, int is_metadata,
251 int have_csum, u8 *csum, u64 generation,
252 u16 csum_size, int retry_failed_mirror);
253 static void scrub_recheck_block_checksum(struct btrfs_fs_info *fs_info,
254 struct scrub_block *sblock,
255 int is_metadata, int have_csum,
256 const u8 *csum, u64 generation,
257 u16 csum_size);
258 static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
259 struct scrub_block *sblock_good);
260 static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
261 struct scrub_block *sblock_good,
262 int page_num, int force_write);
263 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock);
264 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
265 int page_num);
266 static int scrub_checksum_data(struct scrub_block *sblock);
267 static int scrub_checksum_tree_block(struct scrub_block *sblock);
268 static int scrub_checksum_super(struct scrub_block *sblock);
269 static void scrub_block_get(struct scrub_block *sblock);
270 static void scrub_block_put(struct scrub_block *sblock);
271 static void scrub_page_get(struct scrub_page *spage);
272 static void scrub_page_put(struct scrub_page *spage);
273 static void scrub_parity_get(struct scrub_parity *sparity);
274 static void scrub_parity_put(struct scrub_parity *sparity);
275 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
276 struct scrub_page *spage);
277 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
278 u64 physical, struct btrfs_device *dev, u64 flags,
279 u64 gen, int mirror_num, u8 *csum, int force,
280 u64 physical_for_dev_replace);
281 static void scrub_bio_end_io(struct bio *bio, int err);
282 static void scrub_bio_end_io_worker(struct btrfs_work *work);
283 static void scrub_block_complete(struct scrub_block *sblock);
284 static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
285 u64 extent_logical, u64 extent_len,
286 u64 *extent_physical,
287 struct btrfs_device **extent_dev,
288 int *extent_mirror_num);
289 static int scrub_setup_wr_ctx(struct scrub_ctx *sctx,
290 struct scrub_wr_ctx *wr_ctx,
291 struct btrfs_fs_info *fs_info,
292 struct btrfs_device *dev,
293 int is_dev_replace);
294 static void scrub_free_wr_ctx(struct scrub_wr_ctx *wr_ctx);
295 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
296 struct scrub_page *spage);
297 static void scrub_wr_submit(struct scrub_ctx *sctx);
298 static void scrub_wr_bio_end_io(struct bio *bio, int err);
299 static void scrub_wr_bio_end_io_worker(struct btrfs_work *work);
300 static int write_page_nocow(struct scrub_ctx *sctx,
301 u64 physical_for_dev_replace, struct page *page);
302 static int copy_nocow_pages_for_inode(u64 inum, u64 offset, u64 root,
303 struct scrub_copy_nocow_ctx *ctx);
304 static int copy_nocow_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
305 int mirror_num, u64 physical_for_dev_replace);
306 static void copy_nocow_pages_worker(struct btrfs_work *work);
307 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
308 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
309 static void scrub_put_ctx(struct scrub_ctx *sctx);
310
311
312 static void scrub_pending_bio_inc(struct scrub_ctx *sctx)
313 {
314 atomic_inc(&sctx->refs);
315 atomic_inc(&sctx->bios_in_flight);
316 }
317
318 static void scrub_pending_bio_dec(struct scrub_ctx *sctx)
319 {
320 atomic_dec(&sctx->bios_in_flight);
321 wake_up(&sctx->list_wait);
322 scrub_put_ctx(sctx);
323 }
324
325 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
326 {
327 while (atomic_read(&fs_info->scrub_pause_req)) {
328 mutex_unlock(&fs_info->scrub_lock);
329 wait_event(fs_info->scrub_pause_wait,
330 atomic_read(&fs_info->scrub_pause_req) == 0);
331 mutex_lock(&fs_info->scrub_lock);
332 }
333 }
334
335 static void scrub_pause_on(struct btrfs_fs_info *fs_info)
336 {
337 atomic_inc(&fs_info->scrubs_paused);
338 wake_up(&fs_info->scrub_pause_wait);
339 }
340
341 static void scrub_pause_off(struct btrfs_fs_info *fs_info)
342 {
343 mutex_lock(&fs_info->scrub_lock);
344 __scrub_blocked_if_needed(fs_info);
345 atomic_dec(&fs_info->scrubs_paused);
346 mutex_unlock(&fs_info->scrub_lock);
347
348 wake_up(&fs_info->scrub_pause_wait);
349 }
350
351 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
352 {
353 scrub_pause_on(fs_info);
354 scrub_pause_off(fs_info);
355 }
356
357 /*
358 * used for workers that require transaction commits (i.e., for the
359 * NOCOW case)
360 */
361 static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx)
362 {
363 struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
364
365 atomic_inc(&sctx->refs);
366 /*
367 * increment scrubs_running to prevent cancel requests from
368 * completing as long as a worker is running. we must also
369 * increment scrubs_paused to prevent deadlocking on pause
370 * requests used for transactions commits (as the worker uses a
371 * transaction context). it is safe to regard the worker
372 * as paused for all matters practical. effectively, we only
373 * avoid cancellation requests from completing.
374 */
375 mutex_lock(&fs_info->scrub_lock);
376 atomic_inc(&fs_info->scrubs_running);
377 atomic_inc(&fs_info->scrubs_paused);
378 mutex_unlock(&fs_info->scrub_lock);
379
380 /*
381 * check if @scrubs_running=@scrubs_paused condition
382 * inside wait_event() is not an atomic operation.
383 * which means we may inc/dec @scrub_running/paused
384 * at any time. Let's wake up @scrub_pause_wait as
385 * much as we can to let commit transaction blocked less.
386 */
387 wake_up(&fs_info->scrub_pause_wait);
388
389 atomic_inc(&sctx->workers_pending);
390 }
391
392 /* used for workers that require transaction commits */
393 static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx)
394 {
395 struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
396
397 /*
398 * see scrub_pending_trans_workers_inc() why we're pretending
399 * to be paused in the scrub counters
400 */
401 mutex_lock(&fs_info->scrub_lock);
402 atomic_dec(&fs_info->scrubs_running);
403 atomic_dec(&fs_info->scrubs_paused);
404 mutex_unlock(&fs_info->scrub_lock);
405 atomic_dec(&sctx->workers_pending);
406 wake_up(&fs_info->scrub_pause_wait);
407 wake_up(&sctx->list_wait);
408 scrub_put_ctx(sctx);
409 }
410
411 static void scrub_free_csums(struct scrub_ctx *sctx)
412 {
413 while (!list_empty(&sctx->csum_list)) {
414 struct btrfs_ordered_sum *sum;
415 sum = list_first_entry(&sctx->csum_list,
416 struct btrfs_ordered_sum, list);
417 list_del(&sum->list);
418 kfree(sum);
419 }
420 }
421
422 static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx)
423 {
424 int i;
425
426 if (!sctx)
427 return;
428
429 scrub_free_wr_ctx(&sctx->wr_ctx);
430
431 /* this can happen when scrub is cancelled */
432 if (sctx->curr != -1) {
433 struct scrub_bio *sbio = sctx->bios[sctx->curr];
434
435 for (i = 0; i < sbio->page_count; i++) {
436 WARN_ON(!sbio->pagev[i]->page);
437 scrub_block_put(sbio->pagev[i]->sblock);
438 }
439 bio_put(sbio->bio);
440 }
441
442 for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
443 struct scrub_bio *sbio = sctx->bios[i];
444
445 if (!sbio)
446 break;
447 kfree(sbio);
448 }
449
450 scrub_free_csums(sctx);
451 kfree(sctx);
452 }
453
454 static void scrub_put_ctx(struct scrub_ctx *sctx)
455 {
456 if (atomic_dec_and_test(&sctx->refs))
457 scrub_free_ctx(sctx);
458 }
459
460 static noinline_for_stack
461 struct scrub_ctx *scrub_setup_ctx(struct btrfs_device *dev, int is_dev_replace)
462 {
463 struct scrub_ctx *sctx;
464 int i;
465 struct btrfs_fs_info *fs_info = dev->dev_root->fs_info;
466 int pages_per_rd_bio;
467 int ret;
468
469 /*
470 * the setting of pages_per_rd_bio is correct for scrub but might
471 * be wrong for the dev_replace code where we might read from
472 * different devices in the initial huge bios. However, that
473 * code is able to correctly handle the case when adding a page
474 * to a bio fails.
475 */
476 if (dev->bdev)
477 pages_per_rd_bio = min_t(int, SCRUB_PAGES_PER_RD_BIO,
478 bio_get_nr_vecs(dev->bdev));
479 else
480 pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO;
481 sctx = kzalloc(sizeof(*sctx), GFP_NOFS);
482 if (!sctx)
483 goto nomem;
484 atomic_set(&sctx->refs, 1);
485 sctx->is_dev_replace = is_dev_replace;
486 sctx->pages_per_rd_bio = pages_per_rd_bio;
487 sctx->curr = -1;
488 sctx->dev_root = dev->dev_root;
489 for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
490 struct scrub_bio *sbio;
491
492 sbio = kzalloc(sizeof(*sbio), GFP_NOFS);
493 if (!sbio)
494 goto nomem;
495 sctx->bios[i] = sbio;
496
497 sbio->index = i;
498 sbio->sctx = sctx;
499 sbio->page_count = 0;
500 btrfs_init_work(&sbio->work, btrfs_scrub_helper,
501 scrub_bio_end_io_worker, NULL, NULL);
502
503 if (i != SCRUB_BIOS_PER_SCTX - 1)
504 sctx->bios[i]->next_free = i + 1;
505 else
506 sctx->bios[i]->next_free = -1;
507 }
508 sctx->first_free = 0;
509 sctx->nodesize = dev->dev_root->nodesize;
510 sctx->sectorsize = dev->dev_root->sectorsize;
511 atomic_set(&sctx->bios_in_flight, 0);
512 atomic_set(&sctx->workers_pending, 0);
513 atomic_set(&sctx->cancel_req, 0);
514 sctx->csum_size = btrfs_super_csum_size(fs_info->super_copy);
515 INIT_LIST_HEAD(&sctx->csum_list);
516
517 spin_lock_init(&sctx->list_lock);
518 spin_lock_init(&sctx->stat_lock);
519 init_waitqueue_head(&sctx->list_wait);
520
521 ret = scrub_setup_wr_ctx(sctx, &sctx->wr_ctx, fs_info,
522 fs_info->dev_replace.tgtdev, is_dev_replace);
523 if (ret) {
524 scrub_free_ctx(sctx);
525 return ERR_PTR(ret);
526 }
527 return sctx;
528
529 nomem:
530 scrub_free_ctx(sctx);
531 return ERR_PTR(-ENOMEM);
532 }
533
534 static int scrub_print_warning_inode(u64 inum, u64 offset, u64 root,
535 void *warn_ctx)
536 {
537 u64 isize;
538 u32 nlink;
539 int ret;
540 int i;
541 struct extent_buffer *eb;
542 struct btrfs_inode_item *inode_item;
543 struct scrub_warning *swarn = warn_ctx;
544 struct btrfs_fs_info *fs_info = swarn->dev->dev_root->fs_info;
545 struct inode_fs_paths *ipath = NULL;
546 struct btrfs_root *local_root;
547 struct btrfs_key root_key;
548 struct btrfs_key key;
549
550 root_key.objectid = root;
551 root_key.type = BTRFS_ROOT_ITEM_KEY;
552 root_key.offset = (u64)-1;
553 local_root = btrfs_read_fs_root_no_name(fs_info, &root_key);
554 if (IS_ERR(local_root)) {
555 ret = PTR_ERR(local_root);
556 goto err;
557 }
558
559 /*
560 * this makes the path point to (inum INODE_ITEM ioff)
561 */
562 key.objectid = inum;
563 key.type = BTRFS_INODE_ITEM_KEY;
564 key.offset = 0;
565
566 ret = btrfs_search_slot(NULL, local_root, &key, swarn->path, 0, 0);
567 if (ret) {
568 btrfs_release_path(swarn->path);
569 goto err;
570 }
571
572 eb = swarn->path->nodes[0];
573 inode_item = btrfs_item_ptr(eb, swarn->path->slots[0],
574 struct btrfs_inode_item);
575 isize = btrfs_inode_size(eb, inode_item);
576 nlink = btrfs_inode_nlink(eb, inode_item);
577 btrfs_release_path(swarn->path);
578
579 ipath = init_ipath(4096, local_root, swarn->path);
580 if (IS_ERR(ipath)) {
581 ret = PTR_ERR(ipath);
582 ipath = NULL;
583 goto err;
584 }
585 ret = paths_from_inode(inum, ipath);
586
587 if (ret < 0)
588 goto err;
589
590 /*
591 * we deliberately ignore the bit ipath might have been too small to
592 * hold all of the paths here
593 */
594 for (i = 0; i < ipath->fspath->elem_cnt; ++i)
595 printk_in_rcu(KERN_WARNING "BTRFS: %s at logical %llu on dev "
596 "%s, sector %llu, root %llu, inode %llu, offset %llu, "
597 "length %llu, links %u (path: %s)\n", swarn->errstr,
598 swarn->logical, rcu_str_deref(swarn->dev->name),
599 (unsigned long long)swarn->sector, root, inum, offset,
600 min(isize - offset, (u64)PAGE_SIZE), nlink,
601 (char *)(unsigned long)ipath->fspath->val[i]);
602
603 free_ipath(ipath);
604 return 0;
605
606 err:
607 printk_in_rcu(KERN_WARNING "BTRFS: %s at logical %llu on dev "
608 "%s, sector %llu, root %llu, inode %llu, offset %llu: path "
609 "resolving failed with ret=%d\n", swarn->errstr,
610 swarn->logical, rcu_str_deref(swarn->dev->name),
611 (unsigned long long)swarn->sector, root, inum, offset, ret);
612
613 free_ipath(ipath);
614 return 0;
615 }
616
617 static void scrub_print_warning(const char *errstr, struct scrub_block *sblock)
618 {
619 struct btrfs_device *dev;
620 struct btrfs_fs_info *fs_info;
621 struct btrfs_path *path;
622 struct btrfs_key found_key;
623 struct extent_buffer *eb;
624 struct btrfs_extent_item *ei;
625 struct scrub_warning swarn;
626 unsigned long ptr = 0;
627 u64 extent_item_pos;
628 u64 flags = 0;
629 u64 ref_root;
630 u32 item_size;
631 u8 ref_level;
632 int ret;
633
634 WARN_ON(sblock->page_count < 1);
635 dev = sblock->pagev[0]->dev;
636 fs_info = sblock->sctx->dev_root->fs_info;
637
638 path = btrfs_alloc_path();
639 if (!path)
640 return;
641
642 swarn.sector = (sblock->pagev[0]->physical) >> 9;
643 swarn.logical = sblock->pagev[0]->logical;
644 swarn.errstr = errstr;
645 swarn.dev = NULL;
646
647 ret = extent_from_logical(fs_info, swarn.logical, path, &found_key,
648 &flags);
649 if (ret < 0)
650 goto out;
651
652 extent_item_pos = swarn.logical - found_key.objectid;
653 swarn.extent_item_size = found_key.offset;
654
655 eb = path->nodes[0];
656 ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item);
657 item_size = btrfs_item_size_nr(eb, path->slots[0]);
658
659 if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
660 do {
661 ret = tree_backref_for_extent(&ptr, eb, &found_key, ei,
662 item_size, &ref_root,
663 &ref_level);
664 printk_in_rcu(KERN_WARNING
665 "BTRFS: %s at logical %llu on dev %s, "
666 "sector %llu: metadata %s (level %d) in tree "
667 "%llu\n", errstr, swarn.logical,
668 rcu_str_deref(dev->name),
669 (unsigned long long)swarn.sector,
670 ref_level ? "node" : "leaf",
671 ret < 0 ? -1 : ref_level,
672 ret < 0 ? -1 : ref_root);
673 } while (ret != 1);
674 btrfs_release_path(path);
675 } else {
676 btrfs_release_path(path);
677 swarn.path = path;
678 swarn.dev = dev;
679 iterate_extent_inodes(fs_info, found_key.objectid,
680 extent_item_pos, 1,
681 scrub_print_warning_inode, &swarn);
682 }
683
684 out:
685 btrfs_free_path(path);
686 }
687
688 static int scrub_fixup_readpage(u64 inum, u64 offset, u64 root, void *fixup_ctx)
689 {
690 struct page *page = NULL;
691 unsigned long index;
692 struct scrub_fixup_nodatasum *fixup = fixup_ctx;
693 int ret;
694 int corrected = 0;
695 struct btrfs_key key;
696 struct inode *inode = NULL;
697 struct btrfs_fs_info *fs_info;
698 u64 end = offset + PAGE_SIZE - 1;
699 struct btrfs_root *local_root;
700 int srcu_index;
701
702 key.objectid = root;
703 key.type = BTRFS_ROOT_ITEM_KEY;
704 key.offset = (u64)-1;
705
706 fs_info = fixup->root->fs_info;
707 srcu_index = srcu_read_lock(&fs_info->subvol_srcu);
708
709 local_root = btrfs_read_fs_root_no_name(fs_info, &key);
710 if (IS_ERR(local_root)) {
711 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
712 return PTR_ERR(local_root);
713 }
714
715 key.type = BTRFS_INODE_ITEM_KEY;
716 key.objectid = inum;
717 key.offset = 0;
718 inode = btrfs_iget(fs_info->sb, &key, local_root, NULL);
719 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
720 if (IS_ERR(inode))
721 return PTR_ERR(inode);
722
723 index = offset >> PAGE_CACHE_SHIFT;
724
725 page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
726 if (!page) {
727 ret = -ENOMEM;
728 goto out;
729 }
730
731 if (PageUptodate(page)) {
732 if (PageDirty(page)) {
733 /*
734 * we need to write the data to the defect sector. the
735 * data that was in that sector is not in memory,
736 * because the page was modified. we must not write the
737 * modified page to that sector.
738 *
739 * TODO: what could be done here: wait for the delalloc
740 * runner to write out that page (might involve
741 * COW) and see whether the sector is still
742 * referenced afterwards.
743 *
744 * For the meantime, we'll treat this error
745 * incorrectable, although there is a chance that a
746 * later scrub will find the bad sector again and that
747 * there's no dirty page in memory, then.
748 */
749 ret = -EIO;
750 goto out;
751 }
752 ret = repair_io_failure(inode, offset, PAGE_SIZE,
753 fixup->logical, page,
754 offset - page_offset(page),
755 fixup->mirror_num);
756 unlock_page(page);
757 corrected = !ret;
758 } else {
759 /*
760 * we need to get good data first. the general readpage path
761 * will call repair_io_failure for us, we just have to make
762 * sure we read the bad mirror.
763 */
764 ret = set_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
765 EXTENT_DAMAGED, GFP_NOFS);
766 if (ret) {
767 /* set_extent_bits should give proper error */
768 WARN_ON(ret > 0);
769 if (ret > 0)
770 ret = -EFAULT;
771 goto out;
772 }
773
774 ret = extent_read_full_page(&BTRFS_I(inode)->io_tree, page,
775 btrfs_get_extent,
776 fixup->mirror_num);
777 wait_on_page_locked(page);
778
779 corrected = !test_range_bit(&BTRFS_I(inode)->io_tree, offset,
780 end, EXTENT_DAMAGED, 0, NULL);
781 if (!corrected)
782 clear_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
783 EXTENT_DAMAGED, GFP_NOFS);
784 }
785
786 out:
787 if (page)
788 put_page(page);
789
790 iput(inode);
791
792 if (ret < 0)
793 return ret;
794
795 if (ret == 0 && corrected) {
796 /*
797 * we only need to call readpage for one of the inodes belonging
798 * to this extent. so make iterate_extent_inodes stop
799 */
800 return 1;
801 }
802
803 return -EIO;
804 }
805
806 static void scrub_fixup_nodatasum(struct btrfs_work *work)
807 {
808 int ret;
809 struct scrub_fixup_nodatasum *fixup;
810 struct scrub_ctx *sctx;
811 struct btrfs_trans_handle *trans = NULL;
812 struct btrfs_path *path;
813 int uncorrectable = 0;
814
815 fixup = container_of(work, struct scrub_fixup_nodatasum, work);
816 sctx = fixup->sctx;
817
818 path = btrfs_alloc_path();
819 if (!path) {
820 spin_lock(&sctx->stat_lock);
821 ++sctx->stat.malloc_errors;
822 spin_unlock(&sctx->stat_lock);
823 uncorrectable = 1;
824 goto out;
825 }
826
827 trans = btrfs_join_transaction(fixup->root);
828 if (IS_ERR(trans)) {
829 uncorrectable = 1;
830 goto out;
831 }
832
833 /*
834 * the idea is to trigger a regular read through the standard path. we
835 * read a page from the (failed) logical address by specifying the
836 * corresponding copynum of the failed sector. thus, that readpage is
837 * expected to fail.
838 * that is the point where on-the-fly error correction will kick in
839 * (once it's finished) and rewrite the failed sector if a good copy
840 * can be found.
841 */
842 ret = iterate_inodes_from_logical(fixup->logical, fixup->root->fs_info,
843 path, scrub_fixup_readpage,
844 fixup);
845 if (ret < 0) {
846 uncorrectable = 1;
847 goto out;
848 }
849 WARN_ON(ret != 1);
850
851 spin_lock(&sctx->stat_lock);
852 ++sctx->stat.corrected_errors;
853 spin_unlock(&sctx->stat_lock);
854
855 out:
856 if (trans && !IS_ERR(trans))
857 btrfs_end_transaction(trans, fixup->root);
858 if (uncorrectable) {
859 spin_lock(&sctx->stat_lock);
860 ++sctx->stat.uncorrectable_errors;
861 spin_unlock(&sctx->stat_lock);
862 btrfs_dev_replace_stats_inc(
863 &sctx->dev_root->fs_info->dev_replace.
864 num_uncorrectable_read_errors);
865 printk_ratelimited_in_rcu(KERN_ERR "BTRFS: "
866 "unable to fixup (nodatasum) error at logical %llu on dev %s\n",
867 fixup->logical, rcu_str_deref(fixup->dev->name));
868 }
869
870 btrfs_free_path(path);
871 kfree(fixup);
872
873 scrub_pending_trans_workers_dec(sctx);
874 }
875
876 static inline void scrub_get_recover(struct scrub_recover *recover)
877 {
878 atomic_inc(&recover->refs);
879 }
880
881 static inline void scrub_put_recover(struct scrub_recover *recover)
882 {
883 if (atomic_dec_and_test(&recover->refs)) {
884 btrfs_put_bbio(recover->bbio);
885 kfree(recover);
886 }
887 }
888
889 /*
890 * scrub_handle_errored_block gets called when either verification of the
891 * pages failed or the bio failed to read, e.g. with EIO. In the latter
892 * case, this function handles all pages in the bio, even though only one
893 * may be bad.
894 * The goal of this function is to repair the errored block by using the
895 * contents of one of the mirrors.
896 */
897 static int scrub_handle_errored_block(struct scrub_block *sblock_to_check)
898 {
899 struct scrub_ctx *sctx = sblock_to_check->sctx;
900 struct btrfs_device *dev;
901 struct btrfs_fs_info *fs_info;
902 u64 length;
903 u64 logical;
904 u64 generation;
905 unsigned int failed_mirror_index;
906 unsigned int is_metadata;
907 unsigned int have_csum;
908 u8 *csum;
909 struct scrub_block *sblocks_for_recheck; /* holds one for each mirror */
910 struct scrub_block *sblock_bad;
911 int ret;
912 int mirror_index;
913 int page_num;
914 int success;
915 static DEFINE_RATELIMIT_STATE(_rs, DEFAULT_RATELIMIT_INTERVAL,
916 DEFAULT_RATELIMIT_BURST);
917
918 BUG_ON(sblock_to_check->page_count < 1);
919 fs_info = sctx->dev_root->fs_info;
920 if (sblock_to_check->pagev[0]->flags & BTRFS_EXTENT_FLAG_SUPER) {
921 /*
922 * if we find an error in a super block, we just report it.
923 * They will get written with the next transaction commit
924 * anyway
925 */
926 spin_lock(&sctx->stat_lock);
927 ++sctx->stat.super_errors;
928 spin_unlock(&sctx->stat_lock);
929 return 0;
930 }
931 length = sblock_to_check->page_count * PAGE_SIZE;
932 logical = sblock_to_check->pagev[0]->logical;
933 generation = sblock_to_check->pagev[0]->generation;
934 BUG_ON(sblock_to_check->pagev[0]->mirror_num < 1);
935 failed_mirror_index = sblock_to_check->pagev[0]->mirror_num - 1;
936 is_metadata = !(sblock_to_check->pagev[0]->flags &
937 BTRFS_EXTENT_FLAG_DATA);
938 have_csum = sblock_to_check->pagev[0]->have_csum;
939 csum = sblock_to_check->pagev[0]->csum;
940 dev = sblock_to_check->pagev[0]->dev;
941
942 if (sctx->is_dev_replace && !is_metadata && !have_csum) {
943 sblocks_for_recheck = NULL;
944 goto nodatasum_case;
945 }
946
947 /*
948 * read all mirrors one after the other. This includes to
949 * re-read the extent or metadata block that failed (that was
950 * the cause that this fixup code is called) another time,
951 * page by page this time in order to know which pages
952 * caused I/O errors and which ones are good (for all mirrors).
953 * It is the goal to handle the situation when more than one
954 * mirror contains I/O errors, but the errors do not
955 * overlap, i.e. the data can be repaired by selecting the
956 * pages from those mirrors without I/O error on the
957 * particular pages. One example (with blocks >= 2 * PAGE_SIZE)
958 * would be that mirror #1 has an I/O error on the first page,
959 * the second page is good, and mirror #2 has an I/O error on
960 * the second page, but the first page is good.
961 * Then the first page of the first mirror can be repaired by
962 * taking the first page of the second mirror, and the
963 * second page of the second mirror can be repaired by
964 * copying the contents of the 2nd page of the 1st mirror.
965 * One more note: if the pages of one mirror contain I/O
966 * errors, the checksum cannot be verified. In order to get
967 * the best data for repairing, the first attempt is to find
968 * a mirror without I/O errors and with a validated checksum.
969 * Only if this is not possible, the pages are picked from
970 * mirrors with I/O errors without considering the checksum.
971 * If the latter is the case, at the end, the checksum of the
972 * repaired area is verified in order to correctly maintain
973 * the statistics.
974 */
975
976 sblocks_for_recheck = kcalloc(BTRFS_MAX_MIRRORS,
977 sizeof(*sblocks_for_recheck), GFP_NOFS);
978 if (!sblocks_for_recheck) {
979 spin_lock(&sctx->stat_lock);
980 sctx->stat.malloc_errors++;
981 sctx->stat.read_errors++;
982 sctx->stat.uncorrectable_errors++;
983 spin_unlock(&sctx->stat_lock);
984 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
985 goto out;
986 }
987
988 /* setup the context, map the logical blocks and alloc the pages */
989 ret = scrub_setup_recheck_block(sblock_to_check, sblocks_for_recheck);
990 if (ret) {
991 spin_lock(&sctx->stat_lock);
992 sctx->stat.read_errors++;
993 sctx->stat.uncorrectable_errors++;
994 spin_unlock(&sctx->stat_lock);
995 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
996 goto out;
997 }
998 BUG_ON(failed_mirror_index >= BTRFS_MAX_MIRRORS);
999 sblock_bad = sblocks_for_recheck + failed_mirror_index;
1000
1001 /* build and submit the bios for the failed mirror, check checksums */
1002 scrub_recheck_block(fs_info, sblock_bad, is_metadata, have_csum,
1003 csum, generation, sctx->csum_size, 1);
1004
1005 if (!sblock_bad->header_error && !sblock_bad->checksum_error &&
1006 sblock_bad->no_io_error_seen) {
1007 /*
1008 * the error disappeared after reading page by page, or
1009 * the area was part of a huge bio and other parts of the
1010 * bio caused I/O errors, or the block layer merged several
1011 * read requests into one and the error is caused by a
1012 * different bio (usually one of the two latter cases is
1013 * the cause)
1014 */
1015 spin_lock(&sctx->stat_lock);
1016 sctx->stat.unverified_errors++;
1017 sblock_to_check->data_corrected = 1;
1018 spin_unlock(&sctx->stat_lock);
1019
1020 if (sctx->is_dev_replace)
1021 scrub_write_block_to_dev_replace(sblock_bad);
1022 goto out;
1023 }
1024
1025 if (!sblock_bad->no_io_error_seen) {
1026 spin_lock(&sctx->stat_lock);
1027 sctx->stat.read_errors++;
1028 spin_unlock(&sctx->stat_lock);
1029 if (__ratelimit(&_rs))
1030 scrub_print_warning("i/o error", sblock_to_check);
1031 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
1032 } else if (sblock_bad->checksum_error) {
1033 spin_lock(&sctx->stat_lock);
1034 sctx->stat.csum_errors++;
1035 spin_unlock(&sctx->stat_lock);
1036 if (__ratelimit(&_rs))
1037 scrub_print_warning("checksum error", sblock_to_check);
1038 btrfs_dev_stat_inc_and_print(dev,
1039 BTRFS_DEV_STAT_CORRUPTION_ERRS);
1040 } else if (sblock_bad->header_error) {
1041 spin_lock(&sctx->stat_lock);
1042 sctx->stat.verify_errors++;
1043 spin_unlock(&sctx->stat_lock);
1044 if (__ratelimit(&_rs))
1045 scrub_print_warning("checksum/header error",
1046 sblock_to_check);
1047 if (sblock_bad->generation_error)
1048 btrfs_dev_stat_inc_and_print(dev,
1049 BTRFS_DEV_STAT_GENERATION_ERRS);
1050 else
1051 btrfs_dev_stat_inc_and_print(dev,
1052 BTRFS_DEV_STAT_CORRUPTION_ERRS);
1053 }
1054
1055 if (sctx->readonly) {
1056 ASSERT(!sctx->is_dev_replace);
1057 goto out;
1058 }
1059
1060 if (!is_metadata && !have_csum) {
1061 struct scrub_fixup_nodatasum *fixup_nodatasum;
1062
1063 WARN_ON(sctx->is_dev_replace);
1064
1065 nodatasum_case:
1066
1067 /*
1068 * !is_metadata and !have_csum, this means that the data
1069 * might not be COW'ed, that it might be modified
1070 * concurrently. The general strategy to work on the
1071 * commit root does not help in the case when COW is not
1072 * used.
1073 */
1074 fixup_nodatasum = kzalloc(sizeof(*fixup_nodatasum), GFP_NOFS);
1075 if (!fixup_nodatasum)
1076 goto did_not_correct_error;
1077 fixup_nodatasum->sctx = sctx;
1078 fixup_nodatasum->dev = dev;
1079 fixup_nodatasum->logical = logical;
1080 fixup_nodatasum->root = fs_info->extent_root;
1081 fixup_nodatasum->mirror_num = failed_mirror_index + 1;
1082 scrub_pending_trans_workers_inc(sctx);
1083 btrfs_init_work(&fixup_nodatasum->work, btrfs_scrub_helper,
1084 scrub_fixup_nodatasum, NULL, NULL);
1085 btrfs_queue_work(fs_info->scrub_workers,
1086 &fixup_nodatasum->work);
1087 goto out;
1088 }
1089
1090 /*
1091 * now build and submit the bios for the other mirrors, check
1092 * checksums.
1093 * First try to pick the mirror which is completely without I/O
1094 * errors and also does not have a checksum error.
1095 * If one is found, and if a checksum is present, the full block
1096 * that is known to contain an error is rewritten. Afterwards
1097 * the block is known to be corrected.
1098 * If a mirror is found which is completely correct, and no
1099 * checksum is present, only those pages are rewritten that had
1100 * an I/O error in the block to be repaired, since it cannot be
1101 * determined, which copy of the other pages is better (and it
1102 * could happen otherwise that a correct page would be
1103 * overwritten by a bad one).
1104 */
1105 for (mirror_index = 0;
1106 mirror_index < BTRFS_MAX_MIRRORS &&
1107 sblocks_for_recheck[mirror_index].page_count > 0;
1108 mirror_index++) {
1109 struct scrub_block *sblock_other;
1110
1111 if (mirror_index == failed_mirror_index)
1112 continue;
1113 sblock_other = sblocks_for_recheck + mirror_index;
1114
1115 /* build and submit the bios, check checksums */
1116 scrub_recheck_block(fs_info, sblock_other, is_metadata,
1117 have_csum, csum, generation,
1118 sctx->csum_size, 0);
1119
1120 if (!sblock_other->header_error &&
1121 !sblock_other->checksum_error &&
1122 sblock_other->no_io_error_seen) {
1123 if (sctx->is_dev_replace) {
1124 scrub_write_block_to_dev_replace(sblock_other);
1125 goto corrected_error;
1126 } else {
1127 ret = scrub_repair_block_from_good_copy(
1128 sblock_bad, sblock_other);
1129 if (!ret)
1130 goto corrected_error;
1131 }
1132 }
1133 }
1134
1135 if (sblock_bad->no_io_error_seen && !sctx->is_dev_replace)
1136 goto did_not_correct_error;
1137
1138 /*
1139 * In case of I/O errors in the area that is supposed to be
1140 * repaired, continue by picking good copies of those pages.
1141 * Select the good pages from mirrors to rewrite bad pages from
1142 * the area to fix. Afterwards verify the checksum of the block
1143 * that is supposed to be repaired. This verification step is
1144 * only done for the purpose of statistic counting and for the
1145 * final scrub report, whether errors remain.
1146 * A perfect algorithm could make use of the checksum and try
1147 * all possible combinations of pages from the different mirrors
1148 * until the checksum verification succeeds. For example, when
1149 * the 2nd page of mirror #1 faces I/O errors, and the 2nd page
1150 * of mirror #2 is readable but the final checksum test fails,
1151 * then the 2nd page of mirror #3 could be tried, whether now
1152 * the final checksum succeedes. But this would be a rare
1153 * exception and is therefore not implemented. At least it is
1154 * avoided that the good copy is overwritten.
1155 * A more useful improvement would be to pick the sectors
1156 * without I/O error based on sector sizes (512 bytes on legacy
1157 * disks) instead of on PAGE_SIZE. Then maybe 512 byte of one
1158 * mirror could be repaired by taking 512 byte of a different
1159 * mirror, even if other 512 byte sectors in the same PAGE_SIZE
1160 * area are unreadable.
1161 */
1162 success = 1;
1163 for (page_num = 0; page_num < sblock_bad->page_count;
1164 page_num++) {
1165 struct scrub_page *page_bad = sblock_bad->pagev[page_num];
1166 struct scrub_block *sblock_other = NULL;
1167
1168 /* skip no-io-error page in scrub */
1169 if (!page_bad->io_error && !sctx->is_dev_replace)
1170 continue;
1171
1172 /* try to find no-io-error page in mirrors */
1173 if (page_bad->io_error) {
1174 for (mirror_index = 0;
1175 mirror_index < BTRFS_MAX_MIRRORS &&
1176 sblocks_for_recheck[mirror_index].page_count > 0;
1177 mirror_index++) {
1178 if (!sblocks_for_recheck[mirror_index].
1179 pagev[page_num]->io_error) {
1180 sblock_other = sblocks_for_recheck +
1181 mirror_index;
1182 break;
1183 }
1184 }
1185 if (!sblock_other)
1186 success = 0;
1187 }
1188
1189 if (sctx->is_dev_replace) {
1190 /*
1191 * did not find a mirror to fetch the page
1192 * from. scrub_write_page_to_dev_replace()
1193 * handles this case (page->io_error), by
1194 * filling the block with zeros before
1195 * submitting the write request
1196 */
1197 if (!sblock_other)
1198 sblock_other = sblock_bad;
1199
1200 if (scrub_write_page_to_dev_replace(sblock_other,
1201 page_num) != 0) {
1202 btrfs_dev_replace_stats_inc(
1203 &sctx->dev_root->
1204 fs_info->dev_replace.
1205 num_write_errors);
1206 success = 0;
1207 }
1208 } else if (sblock_other) {
1209 ret = scrub_repair_page_from_good_copy(sblock_bad,
1210 sblock_other,
1211 page_num, 0);
1212 if (0 == ret)
1213 page_bad->io_error = 0;
1214 else
1215 success = 0;
1216 }
1217 }
1218
1219 if (success && !sctx->is_dev_replace) {
1220 if (is_metadata || have_csum) {
1221 /*
1222 * need to verify the checksum now that all
1223 * sectors on disk are repaired (the write
1224 * request for data to be repaired is on its way).
1225 * Just be lazy and use scrub_recheck_block()
1226 * which re-reads the data before the checksum
1227 * is verified, but most likely the data comes out
1228 * of the page cache.
1229 */
1230 scrub_recheck_block(fs_info, sblock_bad,
1231 is_metadata, have_csum, csum,
1232 generation, sctx->csum_size, 1);
1233 if (!sblock_bad->header_error &&
1234 !sblock_bad->checksum_error &&
1235 sblock_bad->no_io_error_seen)
1236 goto corrected_error;
1237 else
1238 goto did_not_correct_error;
1239 } else {
1240 corrected_error:
1241 spin_lock(&sctx->stat_lock);
1242 sctx->stat.corrected_errors++;
1243 sblock_to_check->data_corrected = 1;
1244 spin_unlock(&sctx->stat_lock);
1245 printk_ratelimited_in_rcu(KERN_ERR
1246 "BTRFS: fixed up error at logical %llu on dev %s\n",
1247 logical, rcu_str_deref(dev->name));
1248 }
1249 } else {
1250 did_not_correct_error:
1251 spin_lock(&sctx->stat_lock);
1252 sctx->stat.uncorrectable_errors++;
1253 spin_unlock(&sctx->stat_lock);
1254 printk_ratelimited_in_rcu(KERN_ERR
1255 "BTRFS: unable to fixup (regular) error at logical %llu on dev %s\n",
1256 logical, rcu_str_deref(dev->name));
1257 }
1258
1259 out:
1260 if (sblocks_for_recheck) {
1261 for (mirror_index = 0; mirror_index < BTRFS_MAX_MIRRORS;
1262 mirror_index++) {
1263 struct scrub_block *sblock = sblocks_for_recheck +
1264 mirror_index;
1265 struct scrub_recover *recover;
1266 int page_index;
1267
1268 for (page_index = 0; page_index < sblock->page_count;
1269 page_index++) {
1270 sblock->pagev[page_index]->sblock = NULL;
1271 recover = sblock->pagev[page_index]->recover;
1272 if (recover) {
1273 scrub_put_recover(recover);
1274 sblock->pagev[page_index]->recover =
1275 NULL;
1276 }
1277 scrub_page_put(sblock->pagev[page_index]);
1278 }
1279 }
1280 kfree(sblocks_for_recheck);
1281 }
1282
1283 return 0;
1284 }
1285
1286 static inline int scrub_nr_raid_mirrors(struct btrfs_bio *bbio)
1287 {
1288 if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID5)
1289 return 2;
1290 else if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID6)
1291 return 3;
1292 else
1293 return (int)bbio->num_stripes;
1294 }
1295
1296 static inline void scrub_stripe_index_and_offset(u64 logical, u64 map_type,
1297 u64 *raid_map,
1298 u64 mapped_length,
1299 int nstripes, int mirror,
1300 int *stripe_index,
1301 u64 *stripe_offset)
1302 {
1303 int i;
1304
1305 if (map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
1306 /* RAID5/6 */
1307 for (i = 0; i < nstripes; i++) {
1308 if (raid_map[i] == RAID6_Q_STRIPE ||
1309 raid_map[i] == RAID5_P_STRIPE)
1310 continue;
1311
1312 if (logical >= raid_map[i] &&
1313 logical < raid_map[i] + mapped_length)
1314 break;
1315 }
1316
1317 *stripe_index = i;
1318 *stripe_offset = logical - raid_map[i];
1319 } else {
1320 /* The other RAID type */
1321 *stripe_index = mirror;
1322 *stripe_offset = 0;
1323 }
1324 }
1325
1326 static int scrub_setup_recheck_block(struct scrub_block *original_sblock,
1327 struct scrub_block *sblocks_for_recheck)
1328 {
1329 struct scrub_ctx *sctx = original_sblock->sctx;
1330 struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
1331 u64 length = original_sblock->page_count * PAGE_SIZE;
1332 u64 logical = original_sblock->pagev[0]->logical;
1333 struct scrub_recover *recover;
1334 struct btrfs_bio *bbio;
1335 u64 sublen;
1336 u64 mapped_length;
1337 u64 stripe_offset;
1338 int stripe_index;
1339 int page_index = 0;
1340 int mirror_index;
1341 int nmirrors;
1342 int ret;
1343
1344 /*
1345 * note: the two members refs and outstanding_pages
1346 * are not used (and not set) in the blocks that are used for
1347 * the recheck procedure
1348 */
1349
1350 while (length > 0) {
1351 sublen = min_t(u64, length, PAGE_SIZE);
1352 mapped_length = sublen;
1353 bbio = NULL;
1354
1355 /*
1356 * with a length of PAGE_SIZE, each returned stripe
1357 * represents one mirror
1358 */
1359 ret = btrfs_map_sblock(fs_info, REQ_GET_READ_MIRRORS, logical,
1360 &mapped_length, &bbio, 0, 1);
1361 if (ret || !bbio || mapped_length < sublen) {
1362 btrfs_put_bbio(bbio);
1363 return -EIO;
1364 }
1365
1366 recover = kzalloc(sizeof(struct scrub_recover), GFP_NOFS);
1367 if (!recover) {
1368 btrfs_put_bbio(bbio);
1369 return -ENOMEM;
1370 }
1371
1372 atomic_set(&recover->refs, 1);
1373 recover->bbio = bbio;
1374 recover->map_length = mapped_length;
1375
1376 BUG_ON(page_index >= SCRUB_PAGES_PER_RD_BIO);
1377
1378 nmirrors = min(scrub_nr_raid_mirrors(bbio), BTRFS_MAX_MIRRORS);
1379
1380 for (mirror_index = 0; mirror_index < nmirrors;
1381 mirror_index++) {
1382 struct scrub_block *sblock;
1383 struct scrub_page *page;
1384
1385 sblock = sblocks_for_recheck + mirror_index;
1386 sblock->sctx = sctx;
1387 page = kzalloc(sizeof(*page), GFP_NOFS);
1388 if (!page) {
1389 leave_nomem:
1390 spin_lock(&sctx->stat_lock);
1391 sctx->stat.malloc_errors++;
1392 spin_unlock(&sctx->stat_lock);
1393 scrub_put_recover(recover);
1394 return -ENOMEM;
1395 }
1396 scrub_page_get(page);
1397 sblock->pagev[page_index] = page;
1398 page->logical = logical;
1399
1400 scrub_stripe_index_and_offset(logical,
1401 bbio->map_type,
1402 bbio->raid_map,
1403 mapped_length,
1404 bbio->num_stripes -
1405 bbio->num_tgtdevs,
1406 mirror_index,
1407 &stripe_index,
1408 &stripe_offset);
1409 page->physical = bbio->stripes[stripe_index].physical +
1410 stripe_offset;
1411 page->dev = bbio->stripes[stripe_index].dev;
1412
1413 BUG_ON(page_index >= original_sblock->page_count);
1414 page->physical_for_dev_replace =
1415 original_sblock->pagev[page_index]->
1416 physical_for_dev_replace;
1417 /* for missing devices, dev->bdev is NULL */
1418 page->mirror_num = mirror_index + 1;
1419 sblock->page_count++;
1420 page->page = alloc_page(GFP_NOFS);
1421 if (!page->page)
1422 goto leave_nomem;
1423
1424 scrub_get_recover(recover);
1425 page->recover = recover;
1426 }
1427 scrub_put_recover(recover);
1428 length -= sublen;
1429 logical += sublen;
1430 page_index++;
1431 }
1432
1433 return 0;
1434 }
1435
1436 struct scrub_bio_ret {
1437 struct completion event;
1438 int error;
1439 };
1440
1441 static void scrub_bio_wait_endio(struct bio *bio, int error)
1442 {
1443 struct scrub_bio_ret *ret = bio->bi_private;
1444
1445 ret->error = error;
1446 complete(&ret->event);
1447 }
1448
1449 static inline int scrub_is_page_on_raid56(struct scrub_page *page)
1450 {
1451 return page->recover &&
1452 (page->recover->bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK);
1453 }
1454
1455 static int scrub_submit_raid56_bio_wait(struct btrfs_fs_info *fs_info,
1456 struct bio *bio,
1457 struct scrub_page *page)
1458 {
1459 struct scrub_bio_ret done;
1460 int ret;
1461
1462 init_completion(&done.event);
1463 done.error = 0;
1464 bio->bi_iter.bi_sector = page->logical >> 9;
1465 bio->bi_private = &done;
1466 bio->bi_end_io = scrub_bio_wait_endio;
1467
1468 ret = raid56_parity_recover(fs_info->fs_root, bio, page->recover->bbio,
1469 page->recover->map_length,
1470 page->mirror_num, 0);
1471 if (ret)
1472 return ret;
1473
1474 wait_for_completion(&done.event);
1475 if (done.error)
1476 return -EIO;
1477
1478 return 0;
1479 }
1480
1481 /*
1482 * this function will check the on disk data for checksum errors, header
1483 * errors and read I/O errors. If any I/O errors happen, the exact pages
1484 * which are errored are marked as being bad. The goal is to enable scrub
1485 * to take those pages that are not errored from all the mirrors so that
1486 * the pages that are errored in the just handled mirror can be repaired.
1487 */
1488 static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
1489 struct scrub_block *sblock, int is_metadata,
1490 int have_csum, u8 *csum, u64 generation,
1491 u16 csum_size, int retry_failed_mirror)
1492 {
1493 int page_num;
1494
1495 sblock->no_io_error_seen = 1;
1496 sblock->header_error = 0;
1497 sblock->checksum_error = 0;
1498
1499 for (page_num = 0; page_num < sblock->page_count; page_num++) {
1500 struct bio *bio;
1501 struct scrub_page *page = sblock->pagev[page_num];
1502
1503 if (page->dev->bdev == NULL) {
1504 page->io_error = 1;
1505 sblock->no_io_error_seen = 0;
1506 continue;
1507 }
1508
1509 WARN_ON(!page->page);
1510 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
1511 if (!bio) {
1512 page->io_error = 1;
1513 sblock->no_io_error_seen = 0;
1514 continue;
1515 }
1516 bio->bi_bdev = page->dev->bdev;
1517
1518 bio_add_page(bio, page->page, PAGE_SIZE, 0);
1519 if (!retry_failed_mirror && scrub_is_page_on_raid56(page)) {
1520 if (scrub_submit_raid56_bio_wait(fs_info, bio, page))
1521 sblock->no_io_error_seen = 0;
1522 } else {
1523 bio->bi_iter.bi_sector = page->physical >> 9;
1524
1525 if (btrfsic_submit_bio_wait(READ, bio))
1526 sblock->no_io_error_seen = 0;
1527 }
1528
1529 bio_put(bio);
1530 }
1531
1532 if (sblock->no_io_error_seen)
1533 scrub_recheck_block_checksum(fs_info, sblock, is_metadata,
1534 have_csum, csum, generation,
1535 csum_size);
1536
1537 return;
1538 }
1539
1540 static inline int scrub_check_fsid(u8 fsid[],
1541 struct scrub_page *spage)
1542 {
1543 struct btrfs_fs_devices *fs_devices = spage->dev->fs_devices;
1544 int ret;
1545
1546 ret = memcmp(fsid, fs_devices->fsid, BTRFS_UUID_SIZE);
1547 return !ret;
1548 }
1549
1550 static void scrub_recheck_block_checksum(struct btrfs_fs_info *fs_info,
1551 struct scrub_block *sblock,
1552 int is_metadata, int have_csum,
1553 const u8 *csum, u64 generation,
1554 u16 csum_size)
1555 {
1556 int page_num;
1557 u8 calculated_csum[BTRFS_CSUM_SIZE];
1558 u32 crc = ~(u32)0;
1559 void *mapped_buffer;
1560
1561 WARN_ON(!sblock->pagev[0]->page);
1562 if (is_metadata) {
1563 struct btrfs_header *h;
1564
1565 mapped_buffer = kmap_atomic(sblock->pagev[0]->page);
1566 h = (struct btrfs_header *)mapped_buffer;
1567
1568 if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h) ||
1569 !scrub_check_fsid(h->fsid, sblock->pagev[0]) ||
1570 memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
1571 BTRFS_UUID_SIZE)) {
1572 sblock->header_error = 1;
1573 } else if (generation != btrfs_stack_header_generation(h)) {
1574 sblock->header_error = 1;
1575 sblock->generation_error = 1;
1576 }
1577 csum = h->csum;
1578 } else {
1579 if (!have_csum)
1580 return;
1581
1582 mapped_buffer = kmap_atomic(sblock->pagev[0]->page);
1583 }
1584
1585 for (page_num = 0;;) {
1586 if (page_num == 0 && is_metadata)
1587 crc = btrfs_csum_data(
1588 ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE,
1589 crc, PAGE_SIZE - BTRFS_CSUM_SIZE);
1590 else
1591 crc = btrfs_csum_data(mapped_buffer, crc, PAGE_SIZE);
1592
1593 kunmap_atomic(mapped_buffer);
1594 page_num++;
1595 if (page_num >= sblock->page_count)
1596 break;
1597 WARN_ON(!sblock->pagev[page_num]->page);
1598
1599 mapped_buffer = kmap_atomic(sblock->pagev[page_num]->page);
1600 }
1601
1602 btrfs_csum_final(crc, calculated_csum);
1603 if (memcmp(calculated_csum, csum, csum_size))
1604 sblock->checksum_error = 1;
1605 }
1606
1607 static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
1608 struct scrub_block *sblock_good)
1609 {
1610 int page_num;
1611 int ret = 0;
1612
1613 for (page_num = 0; page_num < sblock_bad->page_count; page_num++) {
1614 int ret_sub;
1615
1616 ret_sub = scrub_repair_page_from_good_copy(sblock_bad,
1617 sblock_good,
1618 page_num, 1);
1619 if (ret_sub)
1620 ret = ret_sub;
1621 }
1622
1623 return ret;
1624 }
1625
1626 static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
1627 struct scrub_block *sblock_good,
1628 int page_num, int force_write)
1629 {
1630 struct scrub_page *page_bad = sblock_bad->pagev[page_num];
1631 struct scrub_page *page_good = sblock_good->pagev[page_num];
1632
1633 BUG_ON(page_bad->page == NULL);
1634 BUG_ON(page_good->page == NULL);
1635 if (force_write || sblock_bad->header_error ||
1636 sblock_bad->checksum_error || page_bad->io_error) {
1637 struct bio *bio;
1638 int ret;
1639
1640 if (!page_bad->dev->bdev) {
1641 printk_ratelimited(KERN_WARNING "BTRFS: "
1642 "scrub_repair_page_from_good_copy(bdev == NULL) "
1643 "is unexpected!\n");
1644 return -EIO;
1645 }
1646
1647 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
1648 if (!bio)
1649 return -EIO;
1650 bio->bi_bdev = page_bad->dev->bdev;
1651 bio->bi_iter.bi_sector = page_bad->physical >> 9;
1652
1653 ret = bio_add_page(bio, page_good->page, PAGE_SIZE, 0);
1654 if (PAGE_SIZE != ret) {
1655 bio_put(bio);
1656 return -EIO;
1657 }
1658
1659 if (btrfsic_submit_bio_wait(WRITE, bio)) {
1660 btrfs_dev_stat_inc_and_print(page_bad->dev,
1661 BTRFS_DEV_STAT_WRITE_ERRS);
1662 btrfs_dev_replace_stats_inc(
1663 &sblock_bad->sctx->dev_root->fs_info->
1664 dev_replace.num_write_errors);
1665 bio_put(bio);
1666 return -EIO;
1667 }
1668 bio_put(bio);
1669 }
1670
1671 return 0;
1672 }
1673
1674 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock)
1675 {
1676 int page_num;
1677
1678 /*
1679 * This block is used for the check of the parity on the source device,
1680 * so the data needn't be written into the destination device.
1681 */
1682 if (sblock->sparity)
1683 return;
1684
1685 for (page_num = 0; page_num < sblock->page_count; page_num++) {
1686 int ret;
1687
1688 ret = scrub_write_page_to_dev_replace(sblock, page_num);
1689 if (ret)
1690 btrfs_dev_replace_stats_inc(
1691 &sblock->sctx->dev_root->fs_info->dev_replace.
1692 num_write_errors);
1693 }
1694 }
1695
1696 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
1697 int page_num)
1698 {
1699 struct scrub_page *spage = sblock->pagev[page_num];
1700
1701 BUG_ON(spage->page == NULL);
1702 if (spage->io_error) {
1703 void *mapped_buffer = kmap_atomic(spage->page);
1704
1705 memset(mapped_buffer, 0, PAGE_CACHE_SIZE);
1706 flush_dcache_page(spage->page);
1707 kunmap_atomic(mapped_buffer);
1708 }
1709 return scrub_add_page_to_wr_bio(sblock->sctx, spage);
1710 }
1711
1712 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
1713 struct scrub_page *spage)
1714 {
1715 struct scrub_wr_ctx *wr_ctx = &sctx->wr_ctx;
1716 struct scrub_bio *sbio;
1717 int ret;
1718
1719 mutex_lock(&wr_ctx->wr_lock);
1720 again:
1721 if (!wr_ctx->wr_curr_bio) {
1722 wr_ctx->wr_curr_bio = kzalloc(sizeof(*wr_ctx->wr_curr_bio),
1723 GFP_NOFS);
1724 if (!wr_ctx->wr_curr_bio) {
1725 mutex_unlock(&wr_ctx->wr_lock);
1726 return -ENOMEM;
1727 }
1728 wr_ctx->wr_curr_bio->sctx = sctx;
1729 wr_ctx->wr_curr_bio->page_count = 0;
1730 }
1731 sbio = wr_ctx->wr_curr_bio;
1732 if (sbio->page_count == 0) {
1733 struct bio *bio;
1734
1735 sbio->physical = spage->physical_for_dev_replace;
1736 sbio->logical = spage->logical;
1737 sbio->dev = wr_ctx->tgtdev;
1738 bio = sbio->bio;
1739 if (!bio) {
1740 bio = btrfs_io_bio_alloc(GFP_NOFS, wr_ctx->pages_per_wr_bio);
1741 if (!bio) {
1742 mutex_unlock(&wr_ctx->wr_lock);
1743 return -ENOMEM;
1744 }
1745 sbio->bio = bio;
1746 }
1747
1748 bio->bi_private = sbio;
1749 bio->bi_end_io = scrub_wr_bio_end_io;
1750 bio->bi_bdev = sbio->dev->bdev;
1751 bio->bi_iter.bi_sector = sbio->physical >> 9;
1752 sbio->err = 0;
1753 } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
1754 spage->physical_for_dev_replace ||
1755 sbio->logical + sbio->page_count * PAGE_SIZE !=
1756 spage->logical) {
1757 scrub_wr_submit(sctx);
1758 goto again;
1759 }
1760
1761 ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
1762 if (ret != PAGE_SIZE) {
1763 if (sbio->page_count < 1) {
1764 bio_put(sbio->bio);
1765 sbio->bio = NULL;
1766 mutex_unlock(&wr_ctx->wr_lock);
1767 return -EIO;
1768 }
1769 scrub_wr_submit(sctx);
1770 goto again;
1771 }
1772
1773 sbio->pagev[sbio->page_count] = spage;
1774 scrub_page_get(spage);
1775 sbio->page_count++;
1776 if (sbio->page_count == wr_ctx->pages_per_wr_bio)
1777 scrub_wr_submit(sctx);
1778 mutex_unlock(&wr_ctx->wr_lock);
1779
1780 return 0;
1781 }
1782
1783 static void scrub_wr_submit(struct scrub_ctx *sctx)
1784 {
1785 struct scrub_wr_ctx *wr_ctx = &sctx->wr_ctx;
1786 struct scrub_bio *sbio;
1787
1788 if (!wr_ctx->wr_curr_bio)
1789 return;
1790
1791 sbio = wr_ctx->wr_curr_bio;
1792 wr_ctx->wr_curr_bio = NULL;
1793 WARN_ON(!sbio->bio->bi_bdev);
1794 scrub_pending_bio_inc(sctx);
1795 /* process all writes in a single worker thread. Then the block layer
1796 * orders the requests before sending them to the driver which
1797 * doubled the write performance on spinning disks when measured
1798 * with Linux 3.5 */
1799 btrfsic_submit_bio(WRITE, sbio->bio);
1800 }
1801
1802 static void scrub_wr_bio_end_io(struct bio *bio, int err)
1803 {
1804 struct scrub_bio *sbio = bio->bi_private;
1805 struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;
1806
1807 sbio->err = err;
1808 sbio->bio = bio;
1809
1810 btrfs_init_work(&sbio->work, btrfs_scrubwrc_helper,
1811 scrub_wr_bio_end_io_worker, NULL, NULL);
1812 btrfs_queue_work(fs_info->scrub_wr_completion_workers, &sbio->work);
1813 }
1814
1815 static void scrub_wr_bio_end_io_worker(struct btrfs_work *work)
1816 {
1817 struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
1818 struct scrub_ctx *sctx = sbio->sctx;
1819 int i;
1820
1821 WARN_ON(sbio->page_count > SCRUB_PAGES_PER_WR_BIO);
1822 if (sbio->err) {
1823 struct btrfs_dev_replace *dev_replace =
1824 &sbio->sctx->dev_root->fs_info->dev_replace;
1825
1826 for (i = 0; i < sbio->page_count; i++) {
1827 struct scrub_page *spage = sbio->pagev[i];
1828
1829 spage->io_error = 1;
1830 btrfs_dev_replace_stats_inc(&dev_replace->
1831 num_write_errors);
1832 }
1833 }
1834
1835 for (i = 0; i < sbio->page_count; i++)
1836 scrub_page_put(sbio->pagev[i]);
1837
1838 bio_put(sbio->bio);
1839 kfree(sbio);
1840 scrub_pending_bio_dec(sctx);
1841 }
1842
1843 static int scrub_checksum(struct scrub_block *sblock)
1844 {
1845 u64 flags;
1846 int ret;
1847
1848 WARN_ON(sblock->page_count < 1);
1849 flags = sblock->pagev[0]->flags;
1850 ret = 0;
1851 if (flags & BTRFS_EXTENT_FLAG_DATA)
1852 ret = scrub_checksum_data(sblock);
1853 else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)
1854 ret = scrub_checksum_tree_block(sblock);
1855 else if (flags & BTRFS_EXTENT_FLAG_SUPER)
1856 (void)scrub_checksum_super(sblock);
1857 else
1858 WARN_ON(1);
1859 if (ret)
1860 scrub_handle_errored_block(sblock);
1861
1862 return ret;
1863 }
1864
1865 static int scrub_checksum_data(struct scrub_block *sblock)
1866 {
1867 struct scrub_ctx *sctx = sblock->sctx;
1868 u8 csum[BTRFS_CSUM_SIZE];
1869 u8 *on_disk_csum;
1870 struct page *page;
1871 void *buffer;
1872 u32 crc = ~(u32)0;
1873 int fail = 0;
1874 u64 len;
1875 int index;
1876
1877 BUG_ON(sblock->page_count < 1);
1878 if (!sblock->pagev[0]->have_csum)
1879 return 0;
1880
1881 on_disk_csum = sblock->pagev[0]->csum;
1882 page = sblock->pagev[0]->page;
1883 buffer = kmap_atomic(page);
1884
1885 len = sctx->sectorsize;
1886 index = 0;
1887 for (;;) {
1888 u64 l = min_t(u64, len, PAGE_SIZE);
1889
1890 crc = btrfs_csum_data(buffer, crc, l);
1891 kunmap_atomic(buffer);
1892 len -= l;
1893 if (len == 0)
1894 break;
1895 index++;
1896 BUG_ON(index >= sblock->page_count);
1897 BUG_ON(!sblock->pagev[index]->page);
1898 page = sblock->pagev[index]->page;
1899 buffer = kmap_atomic(page);
1900 }
1901
1902 btrfs_csum_final(crc, csum);
1903 if (memcmp(csum, on_disk_csum, sctx->csum_size))
1904 fail = 1;
1905
1906 return fail;
1907 }
1908
1909 static int scrub_checksum_tree_block(struct scrub_block *sblock)
1910 {
1911 struct scrub_ctx *sctx = sblock->sctx;
1912 struct btrfs_header *h;
1913 struct btrfs_root *root = sctx->dev_root;
1914 struct btrfs_fs_info *fs_info = root->fs_info;
1915 u8 calculated_csum[BTRFS_CSUM_SIZE];
1916 u8 on_disk_csum[BTRFS_CSUM_SIZE];
1917 struct page *page;
1918 void *mapped_buffer;
1919 u64 mapped_size;
1920 void *p;
1921 u32 crc = ~(u32)0;
1922 int fail = 0;
1923 int crc_fail = 0;
1924 u64 len;
1925 int index;
1926
1927 BUG_ON(sblock->page_count < 1);
1928 page = sblock->pagev[0]->page;
1929 mapped_buffer = kmap_atomic(page);
1930 h = (struct btrfs_header *)mapped_buffer;
1931 memcpy(on_disk_csum, h->csum, sctx->csum_size);
1932
1933 /*
1934 * we don't use the getter functions here, as we
1935 * a) don't have an extent buffer and
1936 * b) the page is already kmapped
1937 */
1938
1939 if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h))
1940 ++fail;
1941
1942 if (sblock->pagev[0]->generation != btrfs_stack_header_generation(h))
1943 ++fail;
1944
1945 if (!scrub_check_fsid(h->fsid, sblock->pagev[0]))
1946 ++fail;
1947
1948 if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
1949 BTRFS_UUID_SIZE))
1950 ++fail;
1951
1952 len = sctx->nodesize - BTRFS_CSUM_SIZE;
1953 mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
1954 p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
1955 index = 0;
1956 for (;;) {
1957 u64 l = min_t(u64, len, mapped_size);
1958
1959 crc = btrfs_csum_data(p, crc, l);
1960 kunmap_atomic(mapped_buffer);
1961 len -= l;
1962 if (len == 0)
1963 break;
1964 index++;
1965 BUG_ON(index >= sblock->page_count);
1966 BUG_ON(!sblock->pagev[index]->page);
1967 page = sblock->pagev[index]->page;
1968 mapped_buffer = kmap_atomic(page);
1969 mapped_size = PAGE_SIZE;
1970 p = mapped_buffer;
1971 }
1972
1973 btrfs_csum_final(crc, calculated_csum);
1974 if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
1975 ++crc_fail;
1976
1977 return fail || crc_fail;
1978 }
1979
1980 static int scrub_checksum_super(struct scrub_block *sblock)
1981 {
1982 struct btrfs_super_block *s;
1983 struct scrub_ctx *sctx = sblock->sctx;
1984 u8 calculated_csum[BTRFS_CSUM_SIZE];
1985 u8 on_disk_csum[BTRFS_CSUM_SIZE];
1986 struct page *page;
1987 void *mapped_buffer;
1988 u64 mapped_size;
1989 void *p;
1990 u32 crc = ~(u32)0;
1991 int fail_gen = 0;
1992 int fail_cor = 0;
1993 u64 len;
1994 int index;
1995
1996 BUG_ON(sblock->page_count < 1);
1997 page = sblock->pagev[0]->page;
1998 mapped_buffer = kmap_atomic(page);
1999 s = (struct btrfs_super_block *)mapped_buffer;
2000 memcpy(on_disk_csum, s->csum, sctx->csum_size);
2001
2002 if (sblock->pagev[0]->logical != btrfs_super_bytenr(s))
2003 ++fail_cor;
2004
2005 if (sblock->pagev[0]->generation != btrfs_super_generation(s))
2006 ++fail_gen;
2007
2008 if (!scrub_check_fsid(s->fsid, sblock->pagev[0]))
2009 ++fail_cor;
2010
2011 len = BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE;
2012 mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
2013 p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
2014 index = 0;
2015 for (;;) {
2016 u64 l = min_t(u64, len, mapped_size);
2017
2018 crc = btrfs_csum_data(p, crc, l);
2019 kunmap_atomic(mapped_buffer);
2020 len -= l;
2021 if (len == 0)
2022 break;
2023 index++;
2024 BUG_ON(index >= sblock->page_count);
2025 BUG_ON(!sblock->pagev[index]->page);
2026 page = sblock->pagev[index]->page;
2027 mapped_buffer = kmap_atomic(page);
2028 mapped_size = PAGE_SIZE;
2029 p = mapped_buffer;
2030 }
2031
2032 btrfs_csum_final(crc, calculated_csum);
2033 if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
2034 ++fail_cor;
2035
2036 if (fail_cor + fail_gen) {
2037 /*
2038 * if we find an error in a super block, we just report it.
2039 * They will get written with the next transaction commit
2040 * anyway
2041 */
2042 spin_lock(&sctx->stat_lock);
2043 ++sctx->stat.super_errors;
2044 spin_unlock(&sctx->stat_lock);
2045 if (fail_cor)
2046 btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
2047 BTRFS_DEV_STAT_CORRUPTION_ERRS);
2048 else
2049 btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
2050 BTRFS_DEV_STAT_GENERATION_ERRS);
2051 }
2052
2053 return fail_cor + fail_gen;
2054 }
2055
2056 static void scrub_block_get(struct scrub_block *sblock)
2057 {
2058 atomic_inc(&sblock->refs);
2059 }
2060
2061 static void scrub_block_put(struct scrub_block *sblock)
2062 {
2063 if (atomic_dec_and_test(&sblock->refs)) {
2064 int i;
2065
2066 if (sblock->sparity)
2067 scrub_parity_put(sblock->sparity);
2068
2069 for (i = 0; i < sblock->page_count; i++)
2070 scrub_page_put(sblock->pagev[i]);
2071 kfree(sblock);
2072 }
2073 }
2074
2075 static void scrub_page_get(struct scrub_page *spage)
2076 {
2077 atomic_inc(&spage->refs);
2078 }
2079
2080 static void scrub_page_put(struct scrub_page *spage)
2081 {
2082 if (atomic_dec_and_test(&spage->refs)) {
2083 if (spage->page)
2084 __free_page(spage->page);
2085 kfree(spage);
2086 }
2087 }
2088
2089 static void scrub_submit(struct scrub_ctx *sctx)
2090 {
2091 struct scrub_bio *sbio;
2092
2093 if (sctx->curr == -1)
2094 return;
2095
2096 sbio = sctx->bios[sctx->curr];
2097 sctx->curr = -1;
2098 scrub_pending_bio_inc(sctx);
2099
2100 if (!sbio->bio->bi_bdev) {
2101 /*
2102 * this case should not happen. If btrfs_map_block() is
2103 * wrong, it could happen for dev-replace operations on
2104 * missing devices when no mirrors are available, but in
2105 * this case it should already fail the mount.
2106 * This case is handled correctly (but _very_ slowly).
2107 */
2108 printk_ratelimited(KERN_WARNING
2109 "BTRFS: scrub_submit(bio bdev == NULL) is unexpected!\n");
2110 bio_endio(sbio->bio, -EIO);
2111 } else {
2112 btrfsic_submit_bio(READ, sbio->bio);
2113 }
2114 }
2115
2116 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
2117 struct scrub_page *spage)
2118 {
2119 struct scrub_block *sblock = spage->sblock;
2120 struct scrub_bio *sbio;
2121 int ret;
2122
2123 again:
2124 /*
2125 * grab a fresh bio or wait for one to become available
2126 */
2127 while (sctx->curr == -1) {
2128 spin_lock(&sctx->list_lock);
2129 sctx->curr = sctx->first_free;
2130 if (sctx->curr != -1) {
2131 sctx->first_free = sctx->bios[sctx->curr]->next_free;
2132 sctx->bios[sctx->curr]->next_free = -1;
2133 sctx->bios[sctx->curr]->page_count = 0;
2134 spin_unlock(&sctx->list_lock);
2135 } else {
2136 spin_unlock(&sctx->list_lock);
2137 wait_event(sctx->list_wait, sctx->first_free != -1);
2138 }
2139 }
2140 sbio = sctx->bios[sctx->curr];
2141 if (sbio->page_count == 0) {
2142 struct bio *bio;
2143
2144 sbio->physical = spage->physical;
2145 sbio->logical = spage->logical;
2146 sbio->dev = spage->dev;
2147 bio = sbio->bio;
2148 if (!bio) {
2149 bio = btrfs_io_bio_alloc(GFP_NOFS, sctx->pages_per_rd_bio);
2150 if (!bio)
2151 return -ENOMEM;
2152 sbio->bio = bio;
2153 }
2154
2155 bio->bi_private = sbio;
2156 bio->bi_end_io = scrub_bio_end_io;
2157 bio->bi_bdev = sbio->dev->bdev;
2158 bio->bi_iter.bi_sector = sbio->physical >> 9;
2159 sbio->err = 0;
2160 } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
2161 spage->physical ||
2162 sbio->logical + sbio->page_count * PAGE_SIZE !=
2163 spage->logical ||
2164 sbio->dev != spage->dev) {
2165 scrub_submit(sctx);
2166 goto again;
2167 }
2168
2169 sbio->pagev[sbio->page_count] = spage;
2170 ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
2171 if (ret != PAGE_SIZE) {
2172 if (sbio->page_count < 1) {
2173 bio_put(sbio->bio);
2174 sbio->bio = NULL;
2175 return -EIO;
2176 }
2177 scrub_submit(sctx);
2178 goto again;
2179 }
2180
2181 scrub_block_get(sblock); /* one for the page added to the bio */
2182 atomic_inc(&sblock->outstanding_pages);
2183 sbio->page_count++;
2184 if (sbio->page_count == sctx->pages_per_rd_bio)
2185 scrub_submit(sctx);
2186
2187 return 0;
2188 }
2189
2190 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
2191 u64 physical, struct btrfs_device *dev, u64 flags,
2192 u64 gen, int mirror_num, u8 *csum, int force,
2193 u64 physical_for_dev_replace)
2194 {
2195 struct scrub_block *sblock;
2196 int index;
2197
2198 sblock = kzalloc(sizeof(*sblock), GFP_NOFS);
2199 if (!sblock) {
2200 spin_lock(&sctx->stat_lock);
2201 sctx->stat.malloc_errors++;
2202 spin_unlock(&sctx->stat_lock);
2203 return -ENOMEM;
2204 }
2205
2206 /* one ref inside this function, plus one for each page added to
2207 * a bio later on */
2208 atomic_set(&sblock->refs, 1);
2209 sblock->sctx = sctx;
2210 sblock->no_io_error_seen = 1;
2211
2212 for (index = 0; len > 0; index++) {
2213 struct scrub_page *spage;
2214 u64 l = min_t(u64, len, PAGE_SIZE);
2215
2216 spage = kzalloc(sizeof(*spage), GFP_NOFS);
2217 if (!spage) {
2218 leave_nomem:
2219 spin_lock(&sctx->stat_lock);
2220 sctx->stat.malloc_errors++;
2221 spin_unlock(&sctx->stat_lock);
2222 scrub_block_put(sblock);
2223 return -ENOMEM;
2224 }
2225 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
2226 scrub_page_get(spage);
2227 sblock->pagev[index] = spage;
2228 spage->sblock = sblock;
2229 spage->dev = dev;
2230 spage->flags = flags;
2231 spage->generation = gen;
2232 spage->logical = logical;
2233 spage->physical = physical;
2234 spage->physical_for_dev_replace = physical_for_dev_replace;
2235 spage->mirror_num = mirror_num;
2236 if (csum) {
2237 spage->have_csum = 1;
2238 memcpy(spage->csum, csum, sctx->csum_size);
2239 } else {
2240 spage->have_csum = 0;
2241 }
2242 sblock->page_count++;
2243 spage->page = alloc_page(GFP_NOFS);
2244 if (!spage->page)
2245 goto leave_nomem;
2246 len -= l;
2247 logical += l;
2248 physical += l;
2249 physical_for_dev_replace += l;
2250 }
2251
2252 WARN_ON(sblock->page_count == 0);
2253 for (index = 0; index < sblock->page_count; index++) {
2254 struct scrub_page *spage = sblock->pagev[index];
2255 int ret;
2256
2257 ret = scrub_add_page_to_rd_bio(sctx, spage);
2258 if (ret) {
2259 scrub_block_put(sblock);
2260 return ret;
2261 }
2262 }
2263
2264 if (force)
2265 scrub_submit(sctx);
2266
2267 /* last one frees, either here or in bio completion for last page */
2268 scrub_block_put(sblock);
2269 return 0;
2270 }
2271
2272 static void scrub_bio_end_io(struct bio *bio, int err)
2273 {
2274 struct scrub_bio *sbio = bio->bi_private;
2275 struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;
2276
2277 sbio->err = err;
2278 sbio->bio = bio;
2279
2280 btrfs_queue_work(fs_info->scrub_workers, &sbio->work);
2281 }
2282
2283 static void scrub_bio_end_io_worker(struct btrfs_work *work)
2284 {
2285 struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
2286 struct scrub_ctx *sctx = sbio->sctx;
2287 int i;
2288
2289 BUG_ON(sbio->page_count > SCRUB_PAGES_PER_RD_BIO);
2290 if (sbio->err) {
2291 for (i = 0; i < sbio->page_count; i++) {
2292 struct scrub_page *spage = sbio->pagev[i];
2293
2294 spage->io_error = 1;
2295 spage->sblock->no_io_error_seen = 0;
2296 }
2297 }
2298
2299 /* now complete the scrub_block items that have all pages completed */
2300 for (i = 0; i < sbio->page_count; i++) {
2301 struct scrub_page *spage = sbio->pagev[i];
2302 struct scrub_block *sblock = spage->sblock;
2303
2304 if (atomic_dec_and_test(&sblock->outstanding_pages))
2305 scrub_block_complete(sblock);
2306 scrub_block_put(sblock);
2307 }
2308
2309 bio_put(sbio->bio);
2310 sbio->bio = NULL;
2311 spin_lock(&sctx->list_lock);
2312 sbio->next_free = sctx->first_free;
2313 sctx->first_free = sbio->index;
2314 spin_unlock(&sctx->list_lock);
2315
2316 if (sctx->is_dev_replace &&
2317 atomic_read(&sctx->wr_ctx.flush_all_writes)) {
2318 mutex_lock(&sctx->wr_ctx.wr_lock);
2319 scrub_wr_submit(sctx);
2320 mutex_unlock(&sctx->wr_ctx.wr_lock);
2321 }
2322
2323 scrub_pending_bio_dec(sctx);
2324 }
2325
2326 static inline void __scrub_mark_bitmap(struct scrub_parity *sparity,
2327 unsigned long *bitmap,
2328 u64 start, u64 len)
2329 {
2330 u32 offset;
2331 int nsectors;
2332 int sectorsize = sparity->sctx->dev_root->sectorsize;
2333
2334 if (len >= sparity->stripe_len) {
2335 bitmap_set(bitmap, 0, sparity->nsectors);
2336 return;
2337 }
2338
2339 start -= sparity->logic_start;
2340 start = div_u64_rem(start, sparity->stripe_len, &offset);
2341 offset /= sectorsize;
2342 nsectors = (int)len / sectorsize;
2343
2344 if (offset + nsectors <= sparity->nsectors) {
2345 bitmap_set(bitmap, offset, nsectors);
2346 return;
2347 }
2348
2349 bitmap_set(bitmap, offset, sparity->nsectors - offset);
2350 bitmap_set(bitmap, 0, nsectors - (sparity->nsectors - offset));
2351 }
2352
2353 static inline void scrub_parity_mark_sectors_error(struct scrub_parity *sparity,
2354 u64 start, u64 len)
2355 {
2356 __scrub_mark_bitmap(sparity, sparity->ebitmap, start, len);
2357 }
2358
2359 static inline void scrub_parity_mark_sectors_data(struct scrub_parity *sparity,
2360 u64 start, u64 len)
2361 {
2362 __scrub_mark_bitmap(sparity, sparity->dbitmap, start, len);
2363 }
2364
2365 static void scrub_block_complete(struct scrub_block *sblock)
2366 {
2367 int corrupted = 0;
2368
2369 if (!sblock->no_io_error_seen) {
2370 corrupted = 1;
2371 scrub_handle_errored_block(sblock);
2372 } else {
2373 /*
2374 * if has checksum error, write via repair mechanism in
2375 * dev replace case, otherwise write here in dev replace
2376 * case.
2377 */
2378 corrupted = scrub_checksum(sblock);
2379 if (!corrupted && sblock->sctx->is_dev_replace)
2380 scrub_write_block_to_dev_replace(sblock);
2381 }
2382
2383 if (sblock->sparity && corrupted && !sblock->data_corrected) {
2384 u64 start = sblock->pagev[0]->logical;
2385 u64 end = sblock->pagev[sblock->page_count - 1]->logical +
2386 PAGE_SIZE;
2387
2388 scrub_parity_mark_sectors_error(sblock->sparity,
2389 start, end - start);
2390 }
2391 }
2392
2393 static int scrub_find_csum(struct scrub_ctx *sctx, u64 logical, u64 len,
2394 u8 *csum)
2395 {
2396 struct btrfs_ordered_sum *sum = NULL;
2397 unsigned long index;
2398 unsigned long num_sectors;
2399
2400 while (!list_empty(&sctx->csum_list)) {
2401 sum = list_first_entry(&sctx->csum_list,
2402 struct btrfs_ordered_sum, list);
2403 if (sum->bytenr > logical)
2404 return 0;
2405 if (sum->bytenr + sum->len > logical)
2406 break;
2407
2408 ++sctx->stat.csum_discards;
2409 list_del(&sum->list);
2410 kfree(sum);
2411 sum = NULL;
2412 }
2413 if (!sum)
2414 return 0;
2415
2416 index = ((u32)(logical - sum->bytenr)) / sctx->sectorsize;
2417 num_sectors = sum->len / sctx->sectorsize;
2418 memcpy(csum, sum->sums + index, sctx->csum_size);
2419 if (index == num_sectors - 1) {
2420 list_del(&sum->list);
2421 kfree(sum);
2422 }
2423 return 1;
2424 }
2425
2426 /* scrub extent tries to collect up to 64 kB for each bio */
2427 static int scrub_extent(struct scrub_ctx *sctx, u64 logical, u64 len,
2428 u64 physical, struct btrfs_device *dev, u64 flags,
2429 u64 gen, int mirror_num, u64 physical_for_dev_replace)
2430 {
2431 int ret;
2432 u8 csum[BTRFS_CSUM_SIZE];
2433 u32 blocksize;
2434
2435 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2436 blocksize = sctx->sectorsize;
2437 spin_lock(&sctx->stat_lock);
2438 sctx->stat.data_extents_scrubbed++;
2439 sctx->stat.data_bytes_scrubbed += len;
2440 spin_unlock(&sctx->stat_lock);
2441 } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
2442 blocksize = sctx->nodesize;
2443 spin_lock(&sctx->stat_lock);
2444 sctx->stat.tree_extents_scrubbed++;
2445 sctx->stat.tree_bytes_scrubbed += len;
2446 spin_unlock(&sctx->stat_lock);
2447 } else {
2448 blocksize = sctx->sectorsize;
2449 WARN_ON(1);
2450 }
2451
2452 while (len) {
2453 u64 l = min_t(u64, len, blocksize);
2454 int have_csum = 0;
2455
2456 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2457 /* push csums to sbio */
2458 have_csum = scrub_find_csum(sctx, logical, l, csum);
2459 if (have_csum == 0)
2460 ++sctx->stat.no_csum;
2461 if (sctx->is_dev_replace && !have_csum) {
2462 ret = copy_nocow_pages(sctx, logical, l,
2463 mirror_num,
2464 physical_for_dev_replace);
2465 goto behind_scrub_pages;
2466 }
2467 }
2468 ret = scrub_pages(sctx, logical, l, physical, dev, flags, gen,
2469 mirror_num, have_csum ? csum : NULL, 0,
2470 physical_for_dev_replace);
2471 behind_scrub_pages:
2472 if (ret)
2473 return ret;
2474 len -= l;
2475 logical += l;
2476 physical += l;
2477 physical_for_dev_replace += l;
2478 }
2479 return 0;
2480 }
2481
2482 static int scrub_pages_for_parity(struct scrub_parity *sparity,
2483 u64 logical, u64 len,
2484 u64 physical, struct btrfs_device *dev,
2485 u64 flags, u64 gen, int mirror_num, u8 *csum)
2486 {
2487 struct scrub_ctx *sctx = sparity->sctx;
2488 struct scrub_block *sblock;
2489 int index;
2490
2491 sblock = kzalloc(sizeof(*sblock), GFP_NOFS);
2492 if (!sblock) {
2493 spin_lock(&sctx->stat_lock);
2494 sctx->stat.malloc_errors++;
2495 spin_unlock(&sctx->stat_lock);
2496 return -ENOMEM;
2497 }
2498
2499 /* one ref inside this function, plus one for each page added to
2500 * a bio later on */
2501 atomic_set(&sblock->refs, 1);
2502 sblock->sctx = sctx;
2503 sblock->no_io_error_seen = 1;
2504 sblock->sparity = sparity;
2505 scrub_parity_get(sparity);
2506
2507 for (index = 0; len > 0; index++) {
2508 struct scrub_page *spage;
2509 u64 l = min_t(u64, len, PAGE_SIZE);
2510
2511 spage = kzalloc(sizeof(*spage), GFP_NOFS);
2512 if (!spage) {
2513 leave_nomem:
2514 spin_lock(&sctx->stat_lock);
2515 sctx->stat.malloc_errors++;
2516 spin_unlock(&sctx->stat_lock);
2517 scrub_block_put(sblock);
2518 return -ENOMEM;
2519 }
2520 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
2521 /* For scrub block */
2522 scrub_page_get(spage);
2523 sblock->pagev[index] = spage;
2524 /* For scrub parity */
2525 scrub_page_get(spage);
2526 list_add_tail(&spage->list, &sparity->spages);
2527 spage->sblock = sblock;
2528 spage->dev = dev;
2529 spage->flags = flags;
2530 spage->generation = gen;
2531 spage->logical = logical;
2532 spage->physical = physical;
2533 spage->mirror_num = mirror_num;
2534 if (csum) {
2535 spage->have_csum = 1;
2536 memcpy(spage->csum, csum, sctx->csum_size);
2537 } else {
2538 spage->have_csum = 0;
2539 }
2540 sblock->page_count++;
2541 spage->page = alloc_page(GFP_NOFS);
2542 if (!spage->page)
2543 goto leave_nomem;
2544 len -= l;
2545 logical += l;
2546 physical += l;
2547 }
2548
2549 WARN_ON(sblock->page_count == 0);
2550 for (index = 0; index < sblock->page_count; index++) {
2551 struct scrub_page *spage = sblock->pagev[index];
2552 int ret;
2553
2554 ret = scrub_add_page_to_rd_bio(sctx, spage);
2555 if (ret) {
2556 scrub_block_put(sblock);
2557 return ret;
2558 }
2559 }
2560
2561 /* last one frees, either here or in bio completion for last page */
2562 scrub_block_put(sblock);
2563 return 0;
2564 }
2565
2566 static int scrub_extent_for_parity(struct scrub_parity *sparity,
2567 u64 logical, u64 len,
2568 u64 physical, struct btrfs_device *dev,
2569 u64 flags, u64 gen, int mirror_num)
2570 {
2571 struct scrub_ctx *sctx = sparity->sctx;
2572 int ret;
2573 u8 csum[BTRFS_CSUM_SIZE];
2574 u32 blocksize;
2575
2576 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2577 blocksize = sctx->sectorsize;
2578 } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
2579 blocksize = sctx->nodesize;
2580 } else {
2581 blocksize = sctx->sectorsize;
2582 WARN_ON(1);
2583 }
2584
2585 while (len) {
2586 u64 l = min_t(u64, len, blocksize);
2587 int have_csum = 0;
2588
2589 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2590 /* push csums to sbio */
2591 have_csum = scrub_find_csum(sctx, logical, l, csum);
2592 if (have_csum == 0)
2593 goto skip;
2594 }
2595 ret = scrub_pages_for_parity(sparity, logical, l, physical, dev,
2596 flags, gen, mirror_num,
2597 have_csum ? csum : NULL);
2598 if (ret)
2599 return ret;
2600 skip:
2601 len -= l;
2602 logical += l;
2603 physical += l;
2604 }
2605 return 0;
2606 }
2607
2608 /*
2609 * Given a physical address, this will calculate it's
2610 * logical offset. if this is a parity stripe, it will return
2611 * the most left data stripe's logical offset.
2612 *
2613 * return 0 if it is a data stripe, 1 means parity stripe.
2614 */
2615 static int get_raid56_logic_offset(u64 physical, int num,
2616 struct map_lookup *map, u64 *offset,
2617 u64 *stripe_start)
2618 {
2619 int i;
2620 int j = 0;
2621 u64 stripe_nr;
2622 u64 last_offset;
2623 u32 stripe_index;
2624 u32 rot;
2625
2626 last_offset = (physical - map->stripes[num].physical) *
2627 nr_data_stripes(map);
2628 if (stripe_start)
2629 *stripe_start = last_offset;
2630
2631 *offset = last_offset;
2632 for (i = 0; i < nr_data_stripes(map); i++) {
2633 *offset = last_offset + i * map->stripe_len;
2634
2635 stripe_nr = div_u64(*offset, map->stripe_len);
2636 stripe_nr = div_u64(stripe_nr, nr_data_stripes(map));
2637
2638 /* Work out the disk rotation on this stripe-set */
2639 stripe_nr = div_u64_rem(stripe_nr, map->num_stripes, &rot);
2640 /* calculate which stripe this data locates */
2641 rot += i;
2642 stripe_index = rot % map->num_stripes;
2643 if (stripe_index == num)
2644 return 0;
2645 if (stripe_index < num)
2646 j++;
2647 }
2648 *offset = last_offset + j * map->stripe_len;
2649 return 1;
2650 }
2651
2652 static void scrub_free_parity(struct scrub_parity *sparity)
2653 {
2654 struct scrub_ctx *sctx = sparity->sctx;
2655 struct scrub_page *curr, *next;
2656 int nbits;
2657
2658 nbits = bitmap_weight(sparity->ebitmap, sparity->nsectors);
2659 if (nbits) {
2660 spin_lock(&sctx->stat_lock);
2661 sctx->stat.read_errors += nbits;
2662 sctx->stat.uncorrectable_errors += nbits;
2663 spin_unlock(&sctx->stat_lock);
2664 }
2665
2666 list_for_each_entry_safe(curr, next, &sparity->spages, list) {
2667 list_del_init(&curr->list);
2668 scrub_page_put(curr);
2669 }
2670
2671 kfree(sparity);
2672 }
2673
2674 static void scrub_parity_bio_endio_worker(struct btrfs_work *work)
2675 {
2676 struct scrub_parity *sparity = container_of(work, struct scrub_parity,
2677 work);
2678 struct scrub_ctx *sctx = sparity->sctx;
2679
2680 scrub_free_parity(sparity);
2681 scrub_pending_bio_dec(sctx);
2682 }
2683
2684 static void scrub_parity_bio_endio(struct bio *bio, int error)
2685 {
2686 struct scrub_parity *sparity = (struct scrub_parity *)bio->bi_private;
2687
2688 if (error)
2689 bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
2690 sparity->nsectors);
2691
2692 bio_put(bio);
2693
2694 btrfs_init_work(&sparity->work, btrfs_scrubparity_helper,
2695 scrub_parity_bio_endio_worker, NULL, NULL);
2696 btrfs_queue_work(sparity->sctx->dev_root->fs_info->scrub_parity_workers,
2697 &sparity->work);
2698 }
2699
2700 static void scrub_parity_check_and_repair(struct scrub_parity *sparity)
2701 {
2702 struct scrub_ctx *sctx = sparity->sctx;
2703 struct bio *bio;
2704 struct btrfs_raid_bio *rbio;
2705 struct scrub_page *spage;
2706 struct btrfs_bio *bbio = NULL;
2707 u64 length;
2708 int ret;
2709
2710 if (!bitmap_andnot(sparity->dbitmap, sparity->dbitmap, sparity->ebitmap,
2711 sparity->nsectors))
2712 goto out;
2713
2714 length = sparity->logic_end - sparity->logic_start;
2715 ret = btrfs_map_sblock(sctx->dev_root->fs_info, WRITE,
2716 sparity->logic_start,
2717 &length, &bbio, 0, 1);
2718 if (ret || !bbio || !bbio->raid_map)
2719 goto bbio_out;
2720
2721 bio = btrfs_io_bio_alloc(GFP_NOFS, 0);
2722 if (!bio)
2723 goto bbio_out;
2724
2725 bio->bi_iter.bi_sector = sparity->logic_start >> 9;
2726 bio->bi_private = sparity;
2727 bio->bi_end_io = scrub_parity_bio_endio;
2728
2729 rbio = raid56_parity_alloc_scrub_rbio(sctx->dev_root, bio, bbio,
2730 length, sparity->scrub_dev,
2731 sparity->dbitmap,
2732 sparity->nsectors);
2733 if (!rbio)
2734 goto rbio_out;
2735
2736 list_for_each_entry(spage, &sparity->spages, list)
2737 raid56_parity_add_scrub_pages(rbio, spage->page,
2738 spage->logical);
2739
2740 scrub_pending_bio_inc(sctx);
2741 raid56_parity_submit_scrub_rbio(rbio);
2742 return;
2743
2744 rbio_out:
2745 bio_put(bio);
2746 bbio_out:
2747 btrfs_put_bbio(bbio);
2748 bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
2749 sparity->nsectors);
2750 spin_lock(&sctx->stat_lock);
2751 sctx->stat.malloc_errors++;
2752 spin_unlock(&sctx->stat_lock);
2753 out:
2754 scrub_free_parity(sparity);
2755 }
2756
2757 static inline int scrub_calc_parity_bitmap_len(int nsectors)
2758 {
2759 return DIV_ROUND_UP(nsectors, BITS_PER_LONG) * (BITS_PER_LONG / 8);
2760 }
2761
2762 static void scrub_parity_get(struct scrub_parity *sparity)
2763 {
2764 atomic_inc(&sparity->refs);
2765 }
2766
2767 static void scrub_parity_put(struct scrub_parity *sparity)
2768 {
2769 if (!atomic_dec_and_test(&sparity->refs))
2770 return;
2771
2772 scrub_parity_check_and_repair(sparity);
2773 }
2774
2775 static noinline_for_stack int scrub_raid56_parity(struct scrub_ctx *sctx,
2776 struct map_lookup *map,
2777 struct btrfs_device *sdev,
2778 struct btrfs_path *path,
2779 u64 logic_start,
2780 u64 logic_end)
2781 {
2782 struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
2783 struct btrfs_root *root = fs_info->extent_root;
2784 struct btrfs_root *csum_root = fs_info->csum_root;
2785 struct btrfs_extent_item *extent;
2786 u64 flags;
2787 int ret;
2788 int slot;
2789 struct extent_buffer *l;
2790 struct btrfs_key key;
2791 u64 generation;
2792 u64 extent_logical;
2793 u64 extent_physical;
2794 u64 extent_len;
2795 struct btrfs_device *extent_dev;
2796 struct scrub_parity *sparity;
2797 int nsectors;
2798 int bitmap_len;
2799 int extent_mirror_num;
2800 int stop_loop = 0;
2801
2802 nsectors = map->stripe_len / root->sectorsize;
2803 bitmap_len = scrub_calc_parity_bitmap_len(nsectors);
2804 sparity = kzalloc(sizeof(struct scrub_parity) + 2 * bitmap_len,
2805 GFP_NOFS);
2806 if (!sparity) {
2807 spin_lock(&sctx->stat_lock);
2808 sctx->stat.malloc_errors++;
2809 spin_unlock(&sctx->stat_lock);
2810 return -ENOMEM;
2811 }
2812
2813 sparity->stripe_len = map->stripe_len;
2814 sparity->nsectors = nsectors;
2815 sparity->sctx = sctx;
2816 sparity->scrub_dev = sdev;
2817 sparity->logic_start = logic_start;
2818 sparity->logic_end = logic_end;
2819 atomic_set(&sparity->refs, 1);
2820 INIT_LIST_HEAD(&sparity->spages);
2821 sparity->dbitmap = sparity->bitmap;
2822 sparity->ebitmap = (void *)sparity->bitmap + bitmap_len;
2823
2824 ret = 0;
2825 while (logic_start < logic_end) {
2826 if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
2827 key.type = BTRFS_METADATA_ITEM_KEY;
2828 else
2829 key.type = BTRFS_EXTENT_ITEM_KEY;
2830 key.objectid = logic_start;
2831 key.offset = (u64)-1;
2832
2833 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2834 if (ret < 0)
2835 goto out;
2836
2837 if (ret > 0) {
2838 ret = btrfs_previous_extent_item(root, path, 0);
2839 if (ret < 0)
2840 goto out;
2841 if (ret > 0) {
2842 btrfs_release_path(path);
2843 ret = btrfs_search_slot(NULL, root, &key,
2844 path, 0, 0);
2845 if (ret < 0)
2846 goto out;
2847 }
2848 }
2849
2850 stop_loop = 0;
2851 while (1) {
2852 u64 bytes;
2853
2854 l = path->nodes[0];
2855 slot = path->slots[0];
2856 if (slot >= btrfs_header_nritems(l)) {
2857 ret = btrfs_next_leaf(root, path);
2858 if (ret == 0)
2859 continue;
2860 if (ret < 0)
2861 goto out;
2862
2863 stop_loop = 1;
2864 break;
2865 }
2866 btrfs_item_key_to_cpu(l, &key, slot);
2867
2868 if (key.type != BTRFS_EXTENT_ITEM_KEY &&
2869 key.type != BTRFS_METADATA_ITEM_KEY)
2870 goto next;
2871
2872 if (key.type == BTRFS_METADATA_ITEM_KEY)
2873 bytes = root->nodesize;
2874 else
2875 bytes = key.offset;
2876
2877 if (key.objectid + bytes <= logic_start)
2878 goto next;
2879
2880 if (key.objectid >= logic_end) {
2881 stop_loop = 1;
2882 break;
2883 }
2884
2885 while (key.objectid >= logic_start + map->stripe_len)
2886 logic_start += map->stripe_len;
2887
2888 extent = btrfs_item_ptr(l, slot,
2889 struct btrfs_extent_item);
2890 flags = btrfs_extent_flags(l, extent);
2891 generation = btrfs_extent_generation(l, extent);
2892
2893 if ((flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) &&
2894 (key.objectid < logic_start ||
2895 key.objectid + bytes >
2896 logic_start + map->stripe_len)) {
2897 btrfs_err(fs_info, "scrub: tree block %llu spanning stripes, ignored. logical=%llu",
2898 key.objectid, logic_start);
2899 goto next;
2900 }
2901 again:
2902 extent_logical = key.objectid;
2903 extent_len = bytes;
2904
2905 if (extent_logical < logic_start) {
2906 extent_len -= logic_start - extent_logical;
2907 extent_logical = logic_start;
2908 }
2909
2910 if (extent_logical + extent_len >
2911 logic_start + map->stripe_len)
2912 extent_len = logic_start + map->stripe_len -
2913 extent_logical;
2914
2915 scrub_parity_mark_sectors_data(sparity, extent_logical,
2916 extent_len);
2917
2918 scrub_remap_extent(fs_info, extent_logical,
2919 extent_len, &extent_physical,
2920 &extent_dev,
2921 &extent_mirror_num);
2922
2923 ret = btrfs_lookup_csums_range(csum_root,
2924 extent_logical,
2925 extent_logical + extent_len - 1,
2926 &sctx->csum_list, 1);
2927 if (ret)
2928 goto out;
2929
2930 ret = scrub_extent_for_parity(sparity, extent_logical,
2931 extent_len,
2932 extent_physical,
2933 extent_dev, flags,
2934 generation,
2935 extent_mirror_num);
2936
2937 scrub_free_csums(sctx);
2938
2939 if (ret)
2940 goto out;
2941
2942 if (extent_logical + extent_len <
2943 key.objectid + bytes) {
2944 logic_start += map->stripe_len;
2945
2946 if (logic_start >= logic_end) {
2947 stop_loop = 1;
2948 break;
2949 }
2950
2951 if (logic_start < key.objectid + bytes) {
2952 cond_resched();
2953 goto again;
2954 }
2955 }
2956 next:
2957 path->slots[0]++;
2958 }
2959
2960 btrfs_release_path(path);
2961
2962 if (stop_loop)
2963 break;
2964
2965 logic_start += map->stripe_len;
2966 }
2967 out:
2968 if (ret < 0)
2969 scrub_parity_mark_sectors_error(sparity, logic_start,
2970 logic_end - logic_start);
2971 scrub_parity_put(sparity);
2972 scrub_submit(sctx);
2973 mutex_lock(&sctx->wr_ctx.wr_lock);
2974 scrub_wr_submit(sctx);
2975 mutex_unlock(&sctx->wr_ctx.wr_lock);
2976
2977 btrfs_release_path(path);
2978 return ret < 0 ? ret : 0;
2979 }
2980
2981 static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx,
2982 struct map_lookup *map,
2983 struct btrfs_device *scrub_dev,
2984 int num, u64 base, u64 length,
2985 int is_dev_replace)
2986 {
2987 struct btrfs_path *path, *ppath;
2988 struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
2989 struct btrfs_root *root = fs_info->extent_root;
2990 struct btrfs_root *csum_root = fs_info->csum_root;
2991 struct btrfs_extent_item *extent;
2992 struct blk_plug plug;
2993 u64 flags;
2994 int ret;
2995 int slot;
2996 u64 nstripes;
2997 struct extent_buffer *l;
2998 struct btrfs_key key;
2999 u64 physical;
3000 u64 logical;
3001 u64 logic_end;
3002 u64 physical_end;
3003 u64 generation;
3004 int mirror_num;
3005 struct reada_control *reada1;
3006 struct reada_control *reada2;
3007 struct btrfs_key key_start;
3008 struct btrfs_key key_end;
3009 u64 increment = map->stripe_len;
3010 u64 offset;
3011 u64 extent_logical;
3012 u64 extent_physical;
3013 u64 extent_len;
3014 u64 stripe_logical;
3015 u64 stripe_end;
3016 struct btrfs_device *extent_dev;
3017 int extent_mirror_num;
3018 int stop_loop = 0;
3019
3020 physical = map->stripes[num].physical;
3021 offset = 0;
3022 nstripes = div_u64(length, map->stripe_len);
3023 if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
3024 offset = map->stripe_len * num;
3025 increment = map->stripe_len * map->num_stripes;
3026 mirror_num = 1;
3027 } else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
3028 int factor = map->num_stripes / map->sub_stripes;
3029 offset = map->stripe_len * (num / map->sub_stripes);
3030 increment = map->stripe_len * factor;
3031 mirror_num = num % map->sub_stripes + 1;
3032 } else if (map->type & BTRFS_BLOCK_GROUP_RAID1) {
3033 increment = map->stripe_len;
3034 mirror_num = num % map->num_stripes + 1;
3035 } else if (map->type & BTRFS_BLOCK_GROUP_DUP) {
3036 increment = map->stripe_len;
3037 mirror_num = num % map->num_stripes + 1;
3038 } else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3039 get_raid56_logic_offset(physical, num, map, &offset, NULL);
3040 increment = map->stripe_len * nr_data_stripes(map);
3041 mirror_num = 1;
3042 } else {
3043 increment = map->stripe_len;
3044 mirror_num = 1;
3045 }
3046
3047 path = btrfs_alloc_path();
3048 if (!path)
3049 return -ENOMEM;
3050
3051 ppath = btrfs_alloc_path();
3052 if (!ppath) {
3053 btrfs_free_path(path);
3054 return -ENOMEM;
3055 }
3056
3057 /*
3058 * work on commit root. The related disk blocks are static as
3059 * long as COW is applied. This means, it is save to rewrite
3060 * them to repair disk errors without any race conditions
3061 */
3062 path->search_commit_root = 1;
3063 path->skip_locking = 1;
3064
3065 ppath->search_commit_root = 1;
3066 ppath->skip_locking = 1;
3067 /*
3068 * trigger the readahead for extent tree csum tree and wait for
3069 * completion. During readahead, the scrub is officially paused
3070 * to not hold off transaction commits
3071 */
3072 logical = base + offset;
3073 physical_end = physical + nstripes * map->stripe_len;
3074 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3075 get_raid56_logic_offset(physical_end, num,
3076 map, &logic_end, NULL);
3077 logic_end += base;
3078 } else {
3079 logic_end = logical + increment * nstripes;
3080 }
3081 wait_event(sctx->list_wait,
3082 atomic_read(&sctx->bios_in_flight) == 0);
3083 scrub_blocked_if_needed(fs_info);
3084
3085 /* FIXME it might be better to start readahead at commit root */
3086 key_start.objectid = logical;
3087 key_start.type = BTRFS_EXTENT_ITEM_KEY;
3088 key_start.offset = (u64)0;
3089 key_end.objectid = logic_end;
3090 key_end.type = BTRFS_METADATA_ITEM_KEY;
3091 key_end.offset = (u64)-1;
3092 reada1 = btrfs_reada_add(root, &key_start, &key_end);
3093
3094 key_start.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
3095 key_start.type = BTRFS_EXTENT_CSUM_KEY;
3096 key_start.offset = logical;
3097 key_end.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
3098 key_end.type = BTRFS_EXTENT_CSUM_KEY;
3099 key_end.offset = logic_end;
3100 reada2 = btrfs_reada_add(csum_root, &key_start, &key_end);
3101
3102 if (!IS_ERR(reada1))
3103 btrfs_reada_wait(reada1);
3104 if (!IS_ERR(reada2))
3105 btrfs_reada_wait(reada2);
3106
3107
3108 /*
3109 * collect all data csums for the stripe to avoid seeking during
3110 * the scrub. This might currently (crc32) end up to be about 1MB
3111 */
3112 blk_start_plug(&plug);
3113
3114 /*
3115 * now find all extents for each stripe and scrub them
3116 */
3117 ret = 0;
3118 while (physical < physical_end) {
3119 /*
3120 * canceled?
3121 */
3122 if (atomic_read(&fs_info->scrub_cancel_req) ||
3123 atomic_read(&sctx->cancel_req)) {
3124 ret = -ECANCELED;
3125 goto out;
3126 }
3127 /*
3128 * check to see if we have to pause
3129 */
3130 if (atomic_read(&fs_info->scrub_pause_req)) {
3131 /* push queued extents */
3132 atomic_set(&sctx->wr_ctx.flush_all_writes, 1);
3133 scrub_submit(sctx);
3134 mutex_lock(&sctx->wr_ctx.wr_lock);
3135 scrub_wr_submit(sctx);
3136 mutex_unlock(&sctx->wr_ctx.wr_lock);
3137 wait_event(sctx->list_wait,
3138 atomic_read(&sctx->bios_in_flight) == 0);
3139 atomic_set(&sctx->wr_ctx.flush_all_writes, 0);
3140 scrub_blocked_if_needed(fs_info);
3141 }
3142
3143 /* for raid56, we skip parity stripe */
3144 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3145 ret = get_raid56_logic_offset(physical, num, map,
3146 &logical,
3147 &stripe_logical);
3148 logical += base;
3149 if (ret) {
3150 stripe_logical += base;
3151 stripe_end = stripe_logical + increment;
3152 ret = scrub_raid56_parity(sctx, map, scrub_dev,
3153 ppath, stripe_logical,
3154 stripe_end);
3155 if (ret)
3156 goto out;
3157 goto skip;
3158 }
3159 }
3160
3161 if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
3162 key.type = BTRFS_METADATA_ITEM_KEY;
3163 else
3164 key.type = BTRFS_EXTENT_ITEM_KEY;
3165 key.objectid = logical;
3166 key.offset = (u64)-1;
3167
3168 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3169 if (ret < 0)
3170 goto out;
3171
3172 if (ret > 0) {
3173 ret = btrfs_previous_extent_item(root, path, 0);
3174 if (ret < 0)
3175 goto out;
3176 if (ret > 0) {
3177 /* there's no smaller item, so stick with the
3178 * larger one */
3179 btrfs_release_path(path);
3180 ret = btrfs_search_slot(NULL, root, &key,
3181 path, 0, 0);
3182 if (ret < 0)
3183 goto out;
3184 }
3185 }
3186
3187 stop_loop = 0;
3188 while (1) {
3189 u64 bytes;
3190
3191 l = path->nodes[0];
3192 slot = path->slots[0];
3193 if (slot >= btrfs_header_nritems(l)) {
3194 ret = btrfs_next_leaf(root, path);
3195 if (ret == 0)
3196 continue;
3197 if (ret < 0)
3198 goto out;
3199
3200 stop_loop = 1;
3201 break;
3202 }
3203 btrfs_item_key_to_cpu(l, &key, slot);
3204
3205 if (key.type != BTRFS_EXTENT_ITEM_KEY &&
3206 key.type != BTRFS_METADATA_ITEM_KEY)
3207 goto next;
3208
3209 if (key.type == BTRFS_METADATA_ITEM_KEY)
3210 bytes = root->nodesize;
3211 else
3212 bytes = key.offset;
3213
3214 if (key.objectid + bytes <= logical)
3215 goto next;
3216
3217 if (key.objectid >= logical + map->stripe_len) {
3218 /* out of this device extent */
3219 if (key.objectid >= logic_end)
3220 stop_loop = 1;
3221 break;
3222 }
3223
3224 extent = btrfs_item_ptr(l, slot,
3225 struct btrfs_extent_item);
3226 flags = btrfs_extent_flags(l, extent);
3227 generation = btrfs_extent_generation(l, extent);
3228
3229 if ((flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) &&
3230 (key.objectid < logical ||
3231 key.objectid + bytes >
3232 logical + map->stripe_len)) {
3233 btrfs_err(fs_info,
3234 "scrub: tree block %llu spanning "
3235 "stripes, ignored. logical=%llu",
3236 key.objectid, logical);
3237 goto next;
3238 }
3239
3240 again:
3241 extent_logical = key.objectid;
3242 extent_len = bytes;
3243
3244 /*
3245 * trim extent to this stripe
3246 */
3247 if (extent_logical < logical) {
3248 extent_len -= logical - extent_logical;
3249 extent_logical = logical;
3250 }
3251 if (extent_logical + extent_len >
3252 logical + map->stripe_len) {
3253 extent_len = logical + map->stripe_len -
3254 extent_logical;
3255 }
3256
3257 extent_physical = extent_logical - logical + physical;
3258 extent_dev = scrub_dev;
3259 extent_mirror_num = mirror_num;
3260 if (is_dev_replace)
3261 scrub_remap_extent(fs_info, extent_logical,
3262 extent_len, &extent_physical,
3263 &extent_dev,
3264 &extent_mirror_num);
3265
3266 ret = btrfs_lookup_csums_range(csum_root,
3267 extent_logical,
3268 extent_logical +
3269 extent_len - 1,
3270 &sctx->csum_list, 1);
3271 if (ret)
3272 goto out;
3273
3274 ret = scrub_extent(sctx, extent_logical, extent_len,
3275 extent_physical, extent_dev, flags,
3276 generation, extent_mirror_num,
3277 extent_logical - logical + physical);
3278
3279 scrub_free_csums(sctx);
3280
3281 if (ret)
3282 goto out;
3283
3284 if (extent_logical + extent_len <
3285 key.objectid + bytes) {
3286 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3287 /*
3288 * loop until we find next data stripe
3289 * or we have finished all stripes.
3290 */
3291 loop:
3292 physical += map->stripe_len;
3293 ret = get_raid56_logic_offset(physical,
3294 num, map, &logical,
3295 &stripe_logical);
3296 logical += base;
3297
3298 if (ret && physical < physical_end) {
3299 stripe_logical += base;
3300 stripe_end = stripe_logical +
3301 increment;
3302 ret = scrub_raid56_parity(sctx,
3303 map, scrub_dev, ppath,
3304 stripe_logical,
3305 stripe_end);
3306 if (ret)
3307 goto out;
3308 goto loop;
3309 }
3310 } else {
3311 physical += map->stripe_len;
3312 logical += increment;
3313 }
3314 if (logical < key.objectid + bytes) {
3315 cond_resched();
3316 goto again;
3317 }
3318
3319 if (physical >= physical_end) {
3320 stop_loop = 1;
3321 break;
3322 }
3323 }
3324 next:
3325 path->slots[0]++;
3326 }
3327 btrfs_release_path(path);
3328 skip:
3329 logical += increment;
3330 physical += map->stripe_len;
3331 spin_lock(&sctx->stat_lock);
3332 if (stop_loop)
3333 sctx->stat.last_physical = map->stripes[num].physical +
3334 length;
3335 else
3336 sctx->stat.last_physical = physical;
3337 spin_unlock(&sctx->stat_lock);
3338 if (stop_loop)
3339 break;
3340 }
3341 out:
3342 /* push queued extents */
3343 scrub_submit(sctx);
3344 mutex_lock(&sctx->wr_ctx.wr_lock);
3345 scrub_wr_submit(sctx);
3346 mutex_unlock(&sctx->wr_ctx.wr_lock);
3347
3348 blk_finish_plug(&plug);
3349 btrfs_free_path(path);
3350 btrfs_free_path(ppath);
3351 return ret < 0 ? ret : 0;
3352 }
3353
3354 static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx,
3355 struct btrfs_device *scrub_dev,
3356 u64 chunk_tree, u64 chunk_objectid,
3357 u64 chunk_offset, u64 length,
3358 u64 dev_offset, int is_dev_replace)
3359 {
3360 struct btrfs_mapping_tree *map_tree =
3361 &sctx->dev_root->fs_info->mapping_tree;
3362 struct map_lookup *map;
3363 struct extent_map *em;
3364 int i;
3365 int ret = 0;
3366
3367 read_lock(&map_tree->map_tree.lock);
3368 em = lookup_extent_mapping(&map_tree->map_tree, chunk_offset, 1);
3369 read_unlock(&map_tree->map_tree.lock);
3370
3371 if (!em)
3372 return -EINVAL;
3373
3374 map = (struct map_lookup *)em->bdev;
3375 if (em->start != chunk_offset)
3376 goto out;
3377
3378 if (em->len < length)
3379 goto out;
3380
3381 for (i = 0; i < map->num_stripes; ++i) {
3382 if (map->stripes[i].dev->bdev == scrub_dev->bdev &&
3383 map->stripes[i].physical == dev_offset) {
3384 ret = scrub_stripe(sctx, map, scrub_dev, i,
3385 chunk_offset, length,
3386 is_dev_replace);
3387 if (ret)
3388 goto out;
3389 }
3390 }
3391 out:
3392 free_extent_map(em);
3393
3394 return ret;
3395 }
3396
3397 static noinline_for_stack
3398 int scrub_enumerate_chunks(struct scrub_ctx *sctx,
3399 struct btrfs_device *scrub_dev, u64 start, u64 end,
3400 int is_dev_replace)
3401 {
3402 struct btrfs_dev_extent *dev_extent = NULL;
3403 struct btrfs_path *path;
3404 struct btrfs_root *root = sctx->dev_root;
3405 struct btrfs_fs_info *fs_info = root->fs_info;
3406 u64 length;
3407 u64 chunk_tree;
3408 u64 chunk_objectid;
3409 u64 chunk_offset;
3410 int ret;
3411 int slot;
3412 struct extent_buffer *l;
3413 struct btrfs_key key;
3414 struct btrfs_key found_key;
3415 struct btrfs_block_group_cache *cache;
3416 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
3417
3418 path = btrfs_alloc_path();
3419 if (!path)
3420 return -ENOMEM;
3421
3422 path->reada = 2;
3423 path->search_commit_root = 1;
3424 path->skip_locking = 1;
3425
3426 key.objectid = scrub_dev->devid;
3427 key.offset = 0ull;
3428 key.type = BTRFS_DEV_EXTENT_KEY;
3429
3430 while (1) {
3431 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3432 if (ret < 0)
3433 break;
3434 if (ret > 0) {
3435 if (path->slots[0] >=
3436 btrfs_header_nritems(path->nodes[0])) {
3437 ret = btrfs_next_leaf(root, path);
3438 if (ret)
3439 break;
3440 }
3441 }
3442
3443 l = path->nodes[0];
3444 slot = path->slots[0];
3445
3446 btrfs_item_key_to_cpu(l, &found_key, slot);
3447
3448 if (found_key.objectid != scrub_dev->devid)
3449 break;
3450
3451 if (found_key.type != BTRFS_DEV_EXTENT_KEY)
3452 break;
3453
3454 if (found_key.offset >= end)
3455 break;
3456
3457 if (found_key.offset < key.offset)
3458 break;
3459
3460 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
3461 length = btrfs_dev_extent_length(l, dev_extent);
3462
3463 if (found_key.offset + length <= start)
3464 goto skip;
3465
3466 chunk_tree = btrfs_dev_extent_chunk_tree(l, dev_extent);
3467 chunk_objectid = btrfs_dev_extent_chunk_objectid(l, dev_extent);
3468 chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);
3469
3470 /*
3471 * get a reference on the corresponding block group to prevent
3472 * the chunk from going away while we scrub it
3473 */
3474 cache = btrfs_lookup_block_group(fs_info, chunk_offset);
3475
3476 /* some chunks are removed but not committed to disk yet,
3477 * continue scrubbing */
3478 if (!cache)
3479 goto skip;
3480
3481 dev_replace->cursor_right = found_key.offset + length;
3482 dev_replace->cursor_left = found_key.offset;
3483 dev_replace->item_needs_writeback = 1;
3484 ret = scrub_chunk(sctx, scrub_dev, chunk_tree, chunk_objectid,
3485 chunk_offset, length, found_key.offset,
3486 is_dev_replace);
3487
3488 /*
3489 * flush, submit all pending read and write bios, afterwards
3490 * wait for them.
3491 * Note that in the dev replace case, a read request causes
3492 * write requests that are submitted in the read completion
3493 * worker. Therefore in the current situation, it is required
3494 * that all write requests are flushed, so that all read and
3495 * write requests are really completed when bios_in_flight
3496 * changes to 0.
3497 */
3498 atomic_set(&sctx->wr_ctx.flush_all_writes, 1);
3499 scrub_submit(sctx);
3500 mutex_lock(&sctx->wr_ctx.wr_lock);
3501 scrub_wr_submit(sctx);
3502 mutex_unlock(&sctx->wr_ctx.wr_lock);
3503
3504 wait_event(sctx->list_wait,
3505 atomic_read(&sctx->bios_in_flight) == 0);
3506 atomic_inc(&fs_info->scrubs_paused);
3507 wake_up(&fs_info->scrub_pause_wait);
3508
3509 /*
3510 * must be called before we decrease @scrub_paused.
3511 * make sure we don't block transaction commit while
3512 * we are waiting pending workers finished.
3513 */
3514 wait_event(sctx->list_wait,
3515 atomic_read(&sctx->workers_pending) == 0);
3516 atomic_set(&sctx->wr_ctx.flush_all_writes, 0);
3517
3518 mutex_lock(&fs_info->scrub_lock);
3519 __scrub_blocked_if_needed(fs_info);
3520 atomic_dec(&fs_info->scrubs_paused);
3521 mutex_unlock(&fs_info->scrub_lock);
3522 wake_up(&fs_info->scrub_pause_wait);
3523
3524 btrfs_put_block_group(cache);
3525 if (ret)
3526 break;
3527 if (is_dev_replace &&
3528 atomic64_read(&dev_replace->num_write_errors) > 0) {
3529 ret = -EIO;
3530 break;
3531 }
3532 if (sctx->stat.malloc_errors > 0) {
3533 ret = -ENOMEM;
3534 break;
3535 }
3536
3537 dev_replace->cursor_left = dev_replace->cursor_right;
3538 dev_replace->item_needs_writeback = 1;
3539 skip:
3540 key.offset = found_key.offset + length;
3541 btrfs_release_path(path);
3542 }
3543
3544 btrfs_free_path(path);
3545
3546 /*
3547 * ret can still be 1 from search_slot or next_leaf,
3548 * that's not an error
3549 */
3550 return ret < 0 ? ret : 0;
3551 }
3552
3553 static noinline_for_stack int scrub_supers(struct scrub_ctx *sctx,
3554 struct btrfs_device *scrub_dev)
3555 {
3556 int i;
3557 u64 bytenr;
3558 u64 gen;
3559 int ret;
3560 struct btrfs_root *root = sctx->dev_root;
3561
3562 if (test_bit(BTRFS_FS_STATE_ERROR, &root->fs_info->fs_state))
3563 return -EIO;
3564
3565 /* Seed devices of a new filesystem has their own generation. */
3566 if (scrub_dev->fs_devices != root->fs_info->fs_devices)
3567 gen = scrub_dev->generation;
3568 else
3569 gen = root->fs_info->last_trans_committed;
3570
3571 for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
3572 bytenr = btrfs_sb_offset(i);
3573 if (bytenr + BTRFS_SUPER_INFO_SIZE >
3574 scrub_dev->commit_total_bytes)
3575 break;
3576
3577 ret = scrub_pages(sctx, bytenr, BTRFS_SUPER_INFO_SIZE, bytenr,
3578 scrub_dev, BTRFS_EXTENT_FLAG_SUPER, gen, i,
3579 NULL, 1, bytenr);
3580 if (ret)
3581 return ret;
3582 }
3583 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
3584
3585 return 0;
3586 }
3587
3588 /*
3589 * get a reference count on fs_info->scrub_workers. start worker if necessary
3590 */
3591 static noinline_for_stack int scrub_workers_get(struct btrfs_fs_info *fs_info,
3592 int is_dev_replace)
3593 {
3594 unsigned int flags = WQ_FREEZABLE | WQ_UNBOUND;
3595 int max_active = fs_info->thread_pool_size;
3596
3597 if (fs_info->scrub_workers_refcnt == 0) {
3598 if (is_dev_replace)
3599 fs_info->scrub_workers =
3600 btrfs_alloc_workqueue("btrfs-scrub", flags,
3601 1, 4);
3602 else
3603 fs_info->scrub_workers =
3604 btrfs_alloc_workqueue("btrfs-scrub", flags,
3605 max_active, 4);
3606 if (!fs_info->scrub_workers)
3607 goto fail_scrub_workers;
3608
3609 fs_info->scrub_wr_completion_workers =
3610 btrfs_alloc_workqueue("btrfs-scrubwrc", flags,
3611 max_active, 2);
3612 if (!fs_info->scrub_wr_completion_workers)
3613 goto fail_scrub_wr_completion_workers;
3614
3615 fs_info->scrub_nocow_workers =
3616 btrfs_alloc_workqueue("btrfs-scrubnc", flags, 1, 0);
3617 if (!fs_info->scrub_nocow_workers)
3618 goto fail_scrub_nocow_workers;
3619 fs_info->scrub_parity_workers =
3620 btrfs_alloc_workqueue("btrfs-scrubparity", flags,
3621 max_active, 2);
3622 if (!fs_info->scrub_parity_workers)
3623 goto fail_scrub_parity_workers;
3624 }
3625 ++fs_info->scrub_workers_refcnt;
3626 return 0;
3627
3628 fail_scrub_parity_workers:
3629 btrfs_destroy_workqueue(fs_info->scrub_nocow_workers);
3630 fail_scrub_nocow_workers:
3631 btrfs_destroy_workqueue(fs_info->scrub_wr_completion_workers);
3632 fail_scrub_wr_completion_workers:
3633 btrfs_destroy_workqueue(fs_info->scrub_workers);
3634 fail_scrub_workers:
3635 return -ENOMEM;
3636 }
3637
3638 static noinline_for_stack void scrub_workers_put(struct btrfs_fs_info *fs_info)
3639 {
3640 if (--fs_info->scrub_workers_refcnt == 0) {
3641 btrfs_destroy_workqueue(fs_info->scrub_workers);
3642 btrfs_destroy_workqueue(fs_info->scrub_wr_completion_workers);
3643 btrfs_destroy_workqueue(fs_info->scrub_nocow_workers);
3644 btrfs_destroy_workqueue(fs_info->scrub_parity_workers);
3645 }
3646 WARN_ON(fs_info->scrub_workers_refcnt < 0);
3647 }
3648
3649 int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start,
3650 u64 end, struct btrfs_scrub_progress *progress,
3651 int readonly, int is_dev_replace)
3652 {
3653 struct scrub_ctx *sctx;
3654 int ret;
3655 struct btrfs_device *dev;
3656 struct rcu_string *name;
3657
3658 if (btrfs_fs_closing(fs_info))
3659 return -EINVAL;
3660
3661 if (fs_info->chunk_root->nodesize > BTRFS_STRIPE_LEN) {
3662 /*
3663 * in this case scrub is unable to calculate the checksum
3664 * the way scrub is implemented. Do not handle this
3665 * situation at all because it won't ever happen.
3666 */
3667 btrfs_err(fs_info,
3668 "scrub: size assumption nodesize <= BTRFS_STRIPE_LEN (%d <= %d) fails",
3669 fs_info->chunk_root->nodesize, BTRFS_STRIPE_LEN);
3670 return -EINVAL;
3671 }
3672
3673 if (fs_info->chunk_root->sectorsize != PAGE_SIZE) {
3674 /* not supported for data w/o checksums */
3675 btrfs_err(fs_info,
3676 "scrub: size assumption sectorsize != PAGE_SIZE "
3677 "(%d != %lu) fails",
3678 fs_info->chunk_root->sectorsize, PAGE_SIZE);
3679 return -EINVAL;
3680 }
3681
3682 if (fs_info->chunk_root->nodesize >
3683 PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK ||
3684 fs_info->chunk_root->sectorsize >
3685 PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK) {
3686 /*
3687 * would exhaust the array bounds of pagev member in
3688 * struct scrub_block
3689 */
3690 btrfs_err(fs_info, "scrub: size assumption nodesize and sectorsize "
3691 "<= SCRUB_MAX_PAGES_PER_BLOCK (%d <= %d && %d <= %d) fails",
3692 fs_info->chunk_root->nodesize,
3693 SCRUB_MAX_PAGES_PER_BLOCK,
3694 fs_info->chunk_root->sectorsize,
3695 SCRUB_MAX_PAGES_PER_BLOCK);
3696 return -EINVAL;
3697 }
3698
3699
3700 mutex_lock(&fs_info->fs_devices->device_list_mutex);
3701 dev = btrfs_find_device(fs_info, devid, NULL, NULL);
3702 if (!dev || (dev->missing && !is_dev_replace)) {
3703 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3704 return -ENODEV;
3705 }
3706
3707 if (!is_dev_replace && !readonly && !dev->writeable) {
3708 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3709 rcu_read_lock();
3710 name = rcu_dereference(dev->name);
3711 btrfs_err(fs_info, "scrub: device %s is not writable",
3712 name->str);
3713 rcu_read_unlock();
3714 return -EROFS;
3715 }
3716
3717 mutex_lock(&fs_info->scrub_lock);
3718 if (!dev->in_fs_metadata || dev->is_tgtdev_for_dev_replace) {
3719 mutex_unlock(&fs_info->scrub_lock);
3720 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3721 return -EIO;
3722 }
3723
3724 btrfs_dev_replace_lock(&fs_info->dev_replace);
3725 if (dev->scrub_device ||
3726 (!is_dev_replace &&
3727 btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))) {
3728 btrfs_dev_replace_unlock(&fs_info->dev_replace);
3729 mutex_unlock(&fs_info->scrub_lock);
3730 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3731 return -EINPROGRESS;
3732 }
3733 btrfs_dev_replace_unlock(&fs_info->dev_replace);
3734
3735 ret = scrub_workers_get(fs_info, is_dev_replace);
3736 if (ret) {
3737 mutex_unlock(&fs_info->scrub_lock);
3738 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3739 return ret;
3740 }
3741
3742 sctx = scrub_setup_ctx(dev, is_dev_replace);
3743 if (IS_ERR(sctx)) {
3744 mutex_unlock(&fs_info->scrub_lock);
3745 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3746 scrub_workers_put(fs_info);
3747 return PTR_ERR(sctx);
3748 }
3749 sctx->readonly = readonly;
3750 dev->scrub_device = sctx;
3751 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3752
3753 /*
3754 * checking @scrub_pause_req here, we can avoid
3755 * race between committing transaction and scrubbing.
3756 */
3757 __scrub_blocked_if_needed(fs_info);
3758 atomic_inc(&fs_info->scrubs_running);
3759 mutex_unlock(&fs_info->scrub_lock);
3760
3761 if (!is_dev_replace) {
3762 /*
3763 * by holding device list mutex, we can
3764 * kick off writing super in log tree sync.
3765 */
3766 mutex_lock(&fs_info->fs_devices->device_list_mutex);
3767 ret = scrub_supers(sctx, dev);
3768 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3769 }
3770
3771 if (!ret)
3772 ret = scrub_enumerate_chunks(sctx, dev, start, end,
3773 is_dev_replace);
3774
3775 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
3776 atomic_dec(&fs_info->scrubs_running);
3777 wake_up(&fs_info->scrub_pause_wait);
3778
3779 wait_event(sctx->list_wait, atomic_read(&sctx->workers_pending) == 0);
3780
3781 if (progress)
3782 memcpy(progress, &sctx->stat, sizeof(*progress));
3783
3784 mutex_lock(&fs_info->scrub_lock);
3785 dev->scrub_device = NULL;
3786 scrub_workers_put(fs_info);
3787 mutex_unlock(&fs_info->scrub_lock);
3788
3789 scrub_put_ctx(sctx);
3790
3791 return ret;
3792 }
3793
3794 void btrfs_scrub_pause(struct btrfs_root *root)
3795 {
3796 struct btrfs_fs_info *fs_info = root->fs_info;
3797
3798 mutex_lock(&fs_info->scrub_lock);
3799 atomic_inc(&fs_info->scrub_pause_req);
3800 while (atomic_read(&fs_info->scrubs_paused) !=
3801 atomic_read(&fs_info->scrubs_running)) {
3802 mutex_unlock(&fs_info->scrub_lock);
3803 wait_event(fs_info->scrub_pause_wait,
3804 atomic_read(&fs_info->scrubs_paused) ==
3805 atomic_read(&fs_info->scrubs_running));
3806 mutex_lock(&fs_info->scrub_lock);
3807 }
3808 mutex_unlock(&fs_info->scrub_lock);
3809 }
3810
3811 void btrfs_scrub_continue(struct btrfs_root *root)
3812 {
3813 struct btrfs_fs_info *fs_info = root->fs_info;
3814
3815 atomic_dec(&fs_info->scrub_pause_req);
3816 wake_up(&fs_info->scrub_pause_wait);
3817 }
3818
3819 int btrfs_scrub_cancel(struct btrfs_fs_info *fs_info)
3820 {
3821 mutex_lock(&fs_info->scrub_lock);
3822 if (!atomic_read(&fs_info->scrubs_running)) {
3823 mutex_unlock(&fs_info->scrub_lock);
3824 return -ENOTCONN;
3825 }
3826
3827 atomic_inc(&fs_info->scrub_cancel_req);
3828 while (atomic_read(&fs_info->scrubs_running)) {
3829 mutex_unlock(&fs_info->scrub_lock);
3830 wait_event(fs_info->scrub_pause_wait,
3831 atomic_read(&fs_info->scrubs_running) == 0);
3832 mutex_lock(&fs_info->scrub_lock);
3833 }
3834 atomic_dec(&fs_info->scrub_cancel_req);
3835 mutex_unlock(&fs_info->scrub_lock);
3836
3837 return 0;
3838 }
3839
3840 int btrfs_scrub_cancel_dev(struct btrfs_fs_info *fs_info,
3841 struct btrfs_device *dev)
3842 {
3843 struct scrub_ctx *sctx;
3844
3845 mutex_lock(&fs_info->scrub_lock);
3846 sctx = dev->scrub_device;
3847 if (!sctx) {
3848 mutex_unlock(&fs_info->scrub_lock);
3849 return -ENOTCONN;
3850 }
3851 atomic_inc(&sctx->cancel_req);
3852 while (dev->scrub_device) {
3853 mutex_unlock(&fs_info->scrub_lock);
3854 wait_event(fs_info->scrub_pause_wait,
3855 dev->scrub_device == NULL);
3856 mutex_lock(&fs_info->scrub_lock);
3857 }
3858 mutex_unlock(&fs_info->scrub_lock);
3859
3860 return 0;
3861 }
3862
3863 int btrfs_scrub_progress(struct btrfs_root *root, u64 devid,
3864 struct btrfs_scrub_progress *progress)
3865 {
3866 struct btrfs_device *dev;
3867 struct scrub_ctx *sctx = NULL;
3868
3869 mutex_lock(&root->fs_info->fs_devices->device_list_mutex);
3870 dev = btrfs_find_device(root->fs_info, devid, NULL, NULL);
3871 if (dev)
3872 sctx = dev->scrub_device;
3873 if (sctx)
3874 memcpy(progress, &sctx->stat, sizeof(*progress));
3875 mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
3876
3877 return dev ? (sctx ? 0 : -ENOTCONN) : -ENODEV;
3878 }
3879
3880 static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
3881 u64 extent_logical, u64 extent_len,
3882 u64 *extent_physical,
3883 struct btrfs_device **extent_dev,
3884 int *extent_mirror_num)
3885 {
3886 u64 mapped_length;
3887 struct btrfs_bio *bbio = NULL;
3888 int ret;
3889
3890 mapped_length = extent_len;
3891 ret = btrfs_map_block(fs_info, READ, extent_logical,
3892 &mapped_length, &bbio, 0);
3893 if (ret || !bbio || mapped_length < extent_len ||
3894 !bbio->stripes[0].dev->bdev) {
3895 btrfs_put_bbio(bbio);
3896 return;
3897 }
3898
3899 *extent_physical = bbio->stripes[0].physical;
3900 *extent_mirror_num = bbio->mirror_num;
3901 *extent_dev = bbio->stripes[0].dev;
3902 btrfs_put_bbio(bbio);
3903 }
3904
3905 static int scrub_setup_wr_ctx(struct scrub_ctx *sctx,
3906 struct scrub_wr_ctx *wr_ctx,
3907 struct btrfs_fs_info *fs_info,
3908 struct btrfs_device *dev,
3909 int is_dev_replace)
3910 {
3911 WARN_ON(wr_ctx->wr_curr_bio != NULL);
3912
3913 mutex_init(&wr_ctx->wr_lock);
3914 wr_ctx->wr_curr_bio = NULL;
3915 if (!is_dev_replace)
3916 return 0;
3917
3918 WARN_ON(!dev->bdev);
3919 wr_ctx->pages_per_wr_bio = min_t(int, SCRUB_PAGES_PER_WR_BIO,
3920 bio_get_nr_vecs(dev->bdev));
3921 wr_ctx->tgtdev = dev;
3922 atomic_set(&wr_ctx->flush_all_writes, 0);
3923 return 0;
3924 }
3925
3926 static void scrub_free_wr_ctx(struct scrub_wr_ctx *wr_ctx)
3927 {
3928 mutex_lock(&wr_ctx->wr_lock);
3929 kfree(wr_ctx->wr_curr_bio);
3930 wr_ctx->wr_curr_bio = NULL;
3931 mutex_unlock(&wr_ctx->wr_lock);
3932 }
3933
3934 static int copy_nocow_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
3935 int mirror_num, u64 physical_for_dev_replace)
3936 {
3937 struct scrub_copy_nocow_ctx *nocow_ctx;
3938 struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
3939
3940 nocow_ctx = kzalloc(sizeof(*nocow_ctx), GFP_NOFS);
3941 if (!nocow_ctx) {
3942 spin_lock(&sctx->stat_lock);
3943 sctx->stat.malloc_errors++;
3944 spin_unlock(&sctx->stat_lock);
3945 return -ENOMEM;
3946 }
3947
3948 scrub_pending_trans_workers_inc(sctx);
3949
3950 nocow_ctx->sctx = sctx;
3951 nocow_ctx->logical = logical;
3952 nocow_ctx->len = len;
3953 nocow_ctx->mirror_num = mirror_num;
3954 nocow_ctx->physical_for_dev_replace = physical_for_dev_replace;
3955 btrfs_init_work(&nocow_ctx->work, btrfs_scrubnc_helper,
3956 copy_nocow_pages_worker, NULL, NULL);
3957 INIT_LIST_HEAD(&nocow_ctx->inodes);
3958 btrfs_queue_work(fs_info->scrub_nocow_workers,
3959 &nocow_ctx->work);
3960
3961 return 0;
3962 }
3963
3964 static int record_inode_for_nocow(u64 inum, u64 offset, u64 root, void *ctx)
3965 {
3966 struct scrub_copy_nocow_ctx *nocow_ctx = ctx;
3967 struct scrub_nocow_inode *nocow_inode;
3968
3969 nocow_inode = kzalloc(sizeof(*nocow_inode), GFP_NOFS);
3970 if (!nocow_inode)
3971 return -ENOMEM;
3972 nocow_inode->inum = inum;
3973 nocow_inode->offset = offset;
3974 nocow_inode->root = root;
3975 list_add_tail(&nocow_inode->list, &nocow_ctx->inodes);
3976 return 0;
3977 }
3978
3979 #define COPY_COMPLETE 1
3980
3981 static void copy_nocow_pages_worker(struct btrfs_work *work)
3982 {
3983 struct scrub_copy_nocow_ctx *nocow_ctx =
3984 container_of(work, struct scrub_copy_nocow_ctx, work);
3985 struct scrub_ctx *sctx = nocow_ctx->sctx;
3986 u64 logical = nocow_ctx->logical;
3987 u64 len = nocow_ctx->len;
3988 int mirror_num = nocow_ctx->mirror_num;
3989 u64 physical_for_dev_replace = nocow_ctx->physical_for_dev_replace;
3990 int ret;
3991 struct btrfs_trans_handle *trans = NULL;
3992 struct btrfs_fs_info *fs_info;
3993 struct btrfs_path *path;
3994 struct btrfs_root *root;
3995 int not_written = 0;
3996
3997 fs_info = sctx->dev_root->fs_info;
3998 root = fs_info->extent_root;
3999
4000 path = btrfs_alloc_path();
4001 if (!path) {
4002 spin_lock(&sctx->stat_lock);
4003 sctx->stat.malloc_errors++;
4004 spin_unlock(&sctx->stat_lock);
4005 not_written = 1;
4006 goto out;
4007 }
4008
4009 trans = btrfs_join_transaction(root);
4010 if (IS_ERR(trans)) {
4011 not_written = 1;
4012 goto out;
4013 }
4014
4015 ret = iterate_inodes_from_logical(logical, fs_info, path,
4016 record_inode_for_nocow, nocow_ctx);
4017 if (ret != 0 && ret != -ENOENT) {
4018 btrfs_warn(fs_info, "iterate_inodes_from_logical() failed: log %llu, "
4019 "phys %llu, len %llu, mir %u, ret %d",
4020 logical, physical_for_dev_replace, len, mirror_num,
4021 ret);
4022 not_written = 1;
4023 goto out;
4024 }
4025
4026 btrfs_end_transaction(trans, root);
4027 trans = NULL;
4028 while (!list_empty(&nocow_ctx->inodes)) {
4029 struct scrub_nocow_inode *entry;
4030 entry = list_first_entry(&nocow_ctx->inodes,
4031 struct scrub_nocow_inode,
4032 list);
4033 list_del_init(&entry->list);
4034 ret = copy_nocow_pages_for_inode(entry->inum, entry->offset,
4035 entry->root, nocow_ctx);
4036 kfree(entry);
4037 if (ret == COPY_COMPLETE) {
4038 ret = 0;
4039 break;
4040 } else if (ret) {
4041 break;
4042 }
4043 }
4044 out:
4045 while (!list_empty(&nocow_ctx->inodes)) {
4046 struct scrub_nocow_inode *entry;
4047 entry = list_first_entry(&nocow_ctx->inodes,
4048 struct scrub_nocow_inode,
4049 list);
4050 list_del_init(&entry->list);
4051 kfree(entry);
4052 }
4053 if (trans && !IS_ERR(trans))
4054 btrfs_end_transaction(trans, root);
4055 if (not_written)
4056 btrfs_dev_replace_stats_inc(&fs_info->dev_replace.
4057 num_uncorrectable_read_errors);
4058
4059 btrfs_free_path(path);
4060 kfree(nocow_ctx);
4061
4062 scrub_pending_trans_workers_dec(sctx);
4063 }
4064
4065 static int check_extent_to_block(struct inode *inode, u64 start, u64 len,
4066 u64 logical)
4067 {
4068 struct extent_state *cached_state = NULL;
4069 struct btrfs_ordered_extent *ordered;
4070 struct extent_io_tree *io_tree;
4071 struct extent_map *em;
4072 u64 lockstart = start, lockend = start + len - 1;
4073 int ret = 0;
4074
4075 io_tree = &BTRFS_I(inode)->io_tree;
4076
4077 lock_extent_bits(io_tree, lockstart, lockend, 0, &cached_state);
4078 ordered = btrfs_lookup_ordered_range(inode, lockstart, len);
4079 if (ordered) {
4080 btrfs_put_ordered_extent(ordered);
4081 ret = 1;
4082 goto out_unlock;
4083 }
4084
4085 em = btrfs_get_extent(inode, NULL, 0, start, len, 0);
4086 if (IS_ERR(em)) {
4087 ret = PTR_ERR(em);
4088 goto out_unlock;
4089 }
4090
4091 /*
4092 * This extent does not actually cover the logical extent anymore,
4093 * move on to the next inode.
4094 */
4095 if (em->block_start > logical ||
4096 em->block_start + em->block_len < logical + len) {
4097 free_extent_map(em);
4098 ret = 1;
4099 goto out_unlock;
4100 }
4101 free_extent_map(em);
4102
4103 out_unlock:
4104 unlock_extent_cached(io_tree, lockstart, lockend, &cached_state,
4105 GFP_NOFS);
4106 return ret;
4107 }
4108
4109 static int copy_nocow_pages_for_inode(u64 inum, u64 offset, u64 root,
4110 struct scrub_copy_nocow_ctx *nocow_ctx)
4111 {
4112 struct btrfs_fs_info *fs_info = nocow_ctx->sctx->dev_root->fs_info;
4113 struct btrfs_key key;
4114 struct inode *inode;
4115 struct page *page;
4116 struct btrfs_root *local_root;
4117 struct extent_io_tree *io_tree;
4118 u64 physical_for_dev_replace;
4119 u64 nocow_ctx_logical;
4120 u64 len = nocow_ctx->len;
4121 unsigned long index;
4122 int srcu_index;
4123 int ret = 0;
4124 int err = 0;
4125
4126 key.objectid = root;
4127 key.type = BTRFS_ROOT_ITEM_KEY;
4128 key.offset = (u64)-1;
4129
4130 srcu_index = srcu_read_lock(&fs_info->subvol_srcu);
4131
4132 local_root = btrfs_read_fs_root_no_name(fs_info, &key);
4133 if (IS_ERR(local_root)) {
4134 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
4135 return PTR_ERR(local_root);
4136 }
4137
4138 key.type = BTRFS_INODE_ITEM_KEY;
4139 key.objectid = inum;
4140 key.offset = 0;
4141 inode = btrfs_iget(fs_info->sb, &key, local_root, NULL);
4142 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
4143 if (IS_ERR(inode))
4144 return PTR_ERR(inode);
4145
4146 /* Avoid truncate/dio/punch hole.. */
4147 mutex_lock(&inode->i_mutex);
4148 inode_dio_wait(inode);
4149
4150 physical_for_dev_replace = nocow_ctx->physical_for_dev_replace;
4151 io_tree = &BTRFS_I(inode)->io_tree;
4152 nocow_ctx_logical = nocow_ctx->logical;
4153
4154 ret = check_extent_to_block(inode, offset, len, nocow_ctx_logical);
4155 if (ret) {
4156 ret = ret > 0 ? 0 : ret;
4157 goto out;
4158 }
4159
4160 while (len >= PAGE_CACHE_SIZE) {
4161 index = offset >> PAGE_CACHE_SHIFT;
4162 again:
4163 page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
4164 if (!page) {
4165 btrfs_err(fs_info, "find_or_create_page() failed");
4166 ret = -ENOMEM;
4167 goto out;
4168 }
4169
4170 if (PageUptodate(page)) {
4171 if (PageDirty(page))
4172 goto next_page;
4173 } else {
4174 ClearPageError(page);
4175 err = extent_read_full_page(io_tree, page,
4176 btrfs_get_extent,
4177 nocow_ctx->mirror_num);
4178 if (err) {
4179 ret = err;
4180 goto next_page;
4181 }
4182
4183 lock_page(page);
4184 /*
4185 * If the page has been remove from the page cache,
4186 * the data on it is meaningless, because it may be
4187 * old one, the new data may be written into the new
4188 * page in the page cache.
4189 */
4190 if (page->mapping != inode->i_mapping) {
4191 unlock_page(page);
4192 page_cache_release(page);
4193 goto again;
4194 }
4195 if (!PageUptodate(page)) {
4196 ret = -EIO;
4197 goto next_page;
4198 }
4199 }
4200
4201 ret = check_extent_to_block(inode, offset, len,
4202 nocow_ctx_logical);
4203 if (ret) {
4204 ret = ret > 0 ? 0 : ret;
4205 goto next_page;
4206 }
4207
4208 err = write_page_nocow(nocow_ctx->sctx,
4209 physical_for_dev_replace, page);
4210 if (err)
4211 ret = err;
4212 next_page:
4213 unlock_page(page);
4214 page_cache_release(page);
4215
4216 if (ret)
4217 break;
4218
4219 offset += PAGE_CACHE_SIZE;
4220 physical_for_dev_replace += PAGE_CACHE_SIZE;
4221 nocow_ctx_logical += PAGE_CACHE_SIZE;
4222 len -= PAGE_CACHE_SIZE;
4223 }
4224 ret = COPY_COMPLETE;
4225 out:
4226 mutex_unlock(&inode->i_mutex);
4227 iput(inode);
4228 return ret;
4229 }
4230
4231 static int write_page_nocow(struct scrub_ctx *sctx,
4232 u64 physical_for_dev_replace, struct page *page)
4233 {
4234 struct bio *bio;
4235 struct btrfs_device *dev;
4236 int ret;
4237
4238 dev = sctx->wr_ctx.tgtdev;
4239 if (!dev)
4240 return -EIO;
4241 if (!dev->bdev) {
4242 printk_ratelimited(KERN_WARNING
4243 "BTRFS: scrub write_page_nocow(bdev == NULL) is unexpected!\n");
4244 return -EIO;
4245 }
4246 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
4247 if (!bio) {
4248 spin_lock(&sctx->stat_lock);
4249 sctx->stat.malloc_errors++;
4250 spin_unlock(&sctx->stat_lock);
4251 return -ENOMEM;
4252 }
4253 bio->bi_iter.bi_size = 0;
4254 bio->bi_iter.bi_sector = physical_for_dev_replace >> 9;
4255 bio->bi_bdev = dev->bdev;
4256 ret = bio_add_page(bio, page, PAGE_CACHE_SIZE, 0);
4257 if (ret != PAGE_CACHE_SIZE) {
4258 leave_with_eio:
4259 bio_put(bio);
4260 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
4261 return -EIO;
4262 }
4263
4264 if (btrfsic_submit_bio_wait(WRITE_SYNC, bio))
4265 goto leave_with_eio;
4266
4267 bio_put(bio);
4268 return 0;
4269 }
Linux kernel - Deliverables
This page took 0.111053 seconds and 6 git commands to generate.