projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
btrfs: fix clone / extent-same deadlocks
[deliverable/linux.git] / fs / btrfs / scrub.c
CommitLineData
a2de733c 1/*
b6bfebc1 2 * Copyright (C) 2011, 2012 STRATO. All rights reserved.
a2de733c
AJ		 3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
17 */
18
a2de733c 19#include <linux/blkdev.h>
558540c1 20#include <linux/ratelimit.h>
a2de733c
AJ		 21#include "ctree.h"
22#include "volumes.h"
23#include "disk-io.h"
24#include "ordered-data.h"
0ef8e451 25#include "transaction.h"
558540c1 26#include "backref.h"
5da6fcbc 27#include "extent_io.h"
ff023aac 28#include "dev-replace.h"
21adbd5c 29#include "check-integrity.h"
606686ee 30#include "rcu-string.h"
53b381b3 31#include "raid56.h"
a2de733c
AJ		 32
33/*
34 * This is only the first step towards a full-features scrub. It reads all
35 * extent and super block and verifies the checksums. In case a bad checksum
36 * is found or the extent cannot be read, good data will be written back if
37 * any can be found.
38 *
39 * Future enhancements:
a2de733c
AJ		 40 * - In case an unrepairable extent is encountered, track which files are
41 * affected and report them
a2de733c 42 * - track and record media errors, throw out bad devices
a2de733c 43 * - add a mode to also read unallocated space
a2de733c
AJ		 44 */
45
b5d67f64 46struct scrub_block;
d9d181c1 47struct scrub_ctx;
a2de733c 48
ff023aac
SB		 49/*
50 * the following three values only influence the performance.
51 * The last one configures the number of parallel and outstanding I/O
52 * operations. The first two values configure an upper limit for the number
53 * of (dynamically allocated) pages that are added to a bio.
54 */
55#define SCRUB_PAGES_PER_RD_BIO 32 /* 128k per bio */
56#define SCRUB_PAGES_PER_WR_BIO 32 /* 128k per bio */
57#define SCRUB_BIOS_PER_SCTX 64 /* 8MB per device in flight */
7a9e9987
SB		 58
59/*
60 * the following value times PAGE_SIZE needs to be large enough to match the
61 * largest node/leaf/sector size that shall be supported.
62 * Values larger than BTRFS_STRIPE_LEN are not supported.
63 */
b5d67f64 64#define SCRUB_MAX_PAGES_PER_BLOCK 16 /* 64k per node/leaf/sector */
a2de733c 65
af8e2d1d
MX		 66struct scrub_recover {
67 atomic_t refs;
68 struct btrfs_bio *bbio;
af8e2d1d
MX		 69 u64 map_length;
70};
71
a2de733c 72struct scrub_page {
b5d67f64
SB		 73 struct scrub_block *sblock;
74 struct page *page;
442a4f63 75 struct btrfs_device *dev;
5a6ac9ea 76 struct list_head list;
a2de733c
AJ		 77 u64 flags; /* extent flags */
78 u64 generation;
b5d67f64
SB		 79 u64 logical;
80 u64 physical;
ff023aac 81 u64 physical_for_dev_replace;
57019345 82 atomic_t refs;
b5d67f64
SB		 83 struct {
84 unsigned int mirror_num:8;
85 unsigned int have_csum:1;
86 unsigned int io_error:1;
87 };
a2de733c 88 u8 csum[BTRFS_CSUM_SIZE];
af8e2d1d
MX		 89
90 struct scrub_recover *recover;
a2de733c
AJ		 91};
92
93struct scrub_bio {
94 int index;
d9d181c1 95 struct scrub_ctx *sctx;
a36cf8b8 96 struct btrfs_device *dev;
a2de733c
AJ		 97 struct bio *bio;
98 int err;
99 u64 logical;
100 u64 physical;
ff023aac
SB		 101#if SCRUB_PAGES_PER_WR_BIO >= SCRUB_PAGES_PER_RD_BIO
102 struct scrub_page *pagev[SCRUB_PAGES_PER_WR_BIO];
103#else
104 struct scrub_page *pagev[SCRUB_PAGES_PER_RD_BIO];
105#endif
b5d67f64 106 int page_count;
a2de733c
AJ		 107 int next_free;
108 struct btrfs_work work;
109};
110
b5d67f64 111struct scrub_block {
7a9e9987 112 struct scrub_page *pagev[SCRUB_MAX_PAGES_PER_BLOCK];
b5d67f64
SB		 113 int page_count;
114 atomic_t outstanding_pages;
57019345 115 atomic_t refs; /* free mem on transition to zero */
d9d181c1 116 struct scrub_ctx *sctx;
5a6ac9ea 117 struct scrub_parity *sparity;
b5d67f64
SB		 118 struct {
119 unsigned int header_error:1;
120 unsigned int checksum_error:1;
121 unsigned int no_io_error_seen:1;
442a4f63 122 unsigned int generation_error:1; /* also sets header_error */
5a6ac9ea
MX		 123
124 /* The following is for the data used to check parity */
125 /* It is for the data with checksum */
126 unsigned int data_corrected:1;
b5d67f64
SB		 127 };
128};
129
5a6ac9ea
MX		 130/* Used for the chunks with parity stripe such RAID5/6 */
131struct scrub_parity {
132 struct scrub_ctx *sctx;
133
134 struct btrfs_device *scrub_dev;
135
136 u64 logic_start;
137
138 u64 logic_end;
139
140 int nsectors;
141
142 int stripe_len;
143
57019345 144 atomic_t refs;
5a6ac9ea
MX		 145
146 struct list_head spages;
147
148 /* Work of parity check and repair */
149 struct btrfs_work work;
150
151 /* Mark the parity blocks which have data */
152 unsigned long *dbitmap;
153
154 /*
155 * Mark the parity blocks which have data, but errors happen when
156 * read data or check data
157 */
158 unsigned long *ebitmap;
159
160 unsigned long bitmap[0];
161};
162
ff023aac
SB		 163struct scrub_wr_ctx {
164 struct scrub_bio *wr_curr_bio;
165 struct btrfs_device *tgtdev;
166 int pages_per_wr_bio; /* <= SCRUB_PAGES_PER_WR_BIO */
167 atomic_t flush_all_writes;
168 struct mutex wr_lock;
169};
170
d9d181c1 171struct scrub_ctx {
ff023aac 172 struct scrub_bio *bios[SCRUB_BIOS_PER_SCTX];
a36cf8b8 173 struct btrfs_root *dev_root;
a2de733c
AJ		 174 int first_free;
175 int curr;
b6bfebc1
SB		 176 atomic_t bios_in_flight;
177 atomic_t workers_pending;
a2de733c
AJ		 178 spinlock_t list_lock;
179 wait_queue_head_t list_wait;
180 u16 csum_size;
181 struct list_head csum_list;
182 atomic_t cancel_req;
8628764e 183 int readonly;
ff023aac 184 int pages_per_rd_bio;
b5d67f64
SB		 185 u32 sectorsize;
186 u32 nodesize;
63a212ab
SB		 187
188 int is_dev_replace;
ff023aac 189 struct scrub_wr_ctx wr_ctx;
63a212ab 190
a2de733c
AJ		 191 /*
192 * statistics
193 */
194 struct btrfs_scrub_progress stat;
195 spinlock_t stat_lock;
f55985f4
FM		 196
197 /*
198 * Use a ref counter to avoid use-after-free issues. Scrub workers
199 * decrement bios_in_flight and workers_pending and then do a wakeup
200 * on the list_wait wait queue. We must ensure the main scrub task
201 * doesn't free the scrub context before or while the workers are
202 * doing the wakeup() call.
203 */
204 atomic_t refs;
a2de733c
AJ		 205};
206
0ef8e451 207struct scrub_fixup_nodatasum {
d9d181c1 208 struct scrub_ctx *sctx;
a36cf8b8 209 struct btrfs_device *dev;
0ef8e451
JS		 210 u64 logical;
211 struct btrfs_root *root;
212 struct btrfs_work work;
213 int mirror_num;
214};
215
652f25a2
JB		 216struct scrub_nocow_inode {
217 u64 inum;
218 u64 offset;
219 u64 root;
220 struct list_head list;
221};
222
ff023aac
SB		 223struct scrub_copy_nocow_ctx {
224 struct scrub_ctx *sctx;
225 u64 logical;
226 u64 len;
227 int mirror_num;
228 u64 physical_for_dev_replace;
652f25a2 229 struct list_head inodes;
ff023aac
SB		 230 struct btrfs_work work;
231};
232
558540c1
JS		 233struct scrub_warning {
234 struct btrfs_path *path;
235 u64 extent_item_size;
558540c1
JS		 236 const char *errstr;
237 sector_t sector;
238 u64 logical;
239 struct btrfs_device *dev;
558540c1
JS		 240};
241
b6bfebc1
SB		 242static void scrub_pending_bio_inc(struct scrub_ctx *sctx);
243static void scrub_pending_bio_dec(struct scrub_ctx *sctx);
244static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx);
245static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx);
b5d67f64 246static int scrub_handle_errored_block(struct scrub_block *sblock_to_check);
be50a8dd 247static int scrub_setup_recheck_block(struct scrub_block *original_sblock,
ff023aac 248 struct scrub_block *sblocks_for_recheck);
34f5c8e9
SB		 249static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
250 struct scrub_block *sblock, int is_metadata,
251 int have_csum, u8 *csum, u64 generation,
af8e2d1d 252 u16 csum_size, int retry_failed_mirror);
b5d67f64
SB		 253static void scrub_recheck_block_checksum(struct btrfs_fs_info *fs_info,
254 struct scrub_block *sblock,
255 int is_metadata, int have_csum,
256 const u8 *csum, u64 generation,
257 u16 csum_size);
b5d67f64 258static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
114ab50d 259 struct scrub_block *sblock_good);
b5d67f64
SB		 260static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
261 struct scrub_block *sblock_good,
262 int page_num, int force_write);
ff023aac
SB		 263static void scrub_write_block_to_dev_replace(struct scrub_block *sblock);
264static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
265 int page_num);
b5d67f64
SB		 266static int scrub_checksum_data(struct scrub_block *sblock);
267static int scrub_checksum_tree_block(struct scrub_block *sblock);
268static int scrub_checksum_super(struct scrub_block *sblock);
269static void scrub_block_get(struct scrub_block *sblock);
270static void scrub_block_put(struct scrub_block *sblock);
7a9e9987
SB		 271static void scrub_page_get(struct scrub_page *spage);
272static void scrub_page_put(struct scrub_page *spage);
5a6ac9ea
MX		 273static void scrub_parity_get(struct scrub_parity *sparity);
274static void scrub_parity_put(struct scrub_parity *sparity);
ff023aac
SB		 275static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
276 struct scrub_page *spage);
d9d181c1 277static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
a36cf8b8 278 u64 physical, struct btrfs_device *dev, u64 flags,
ff023aac
SB		 279 u64 gen, int mirror_num, u8 *csum, int force,
280 u64 physical_for_dev_replace);
1623edeb 281static void scrub_bio_end_io(struct bio *bio, int err);
b5d67f64
SB		 282static void scrub_bio_end_io_worker(struct btrfs_work *work);
283static void scrub_block_complete(struct scrub_block *sblock);
ff023aac
SB		 284static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
285 u64 extent_logical, u64 extent_len,
286 u64 *extent_physical,
287 struct btrfs_device **extent_dev,
288 int *extent_mirror_num);
289static int scrub_setup_wr_ctx(struct scrub_ctx *sctx,
290 struct scrub_wr_ctx *wr_ctx,
291 struct btrfs_fs_info *fs_info,
292 struct btrfs_device *dev,
293 int is_dev_replace);
294static void scrub_free_wr_ctx(struct scrub_wr_ctx *wr_ctx);
295static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
296 struct scrub_page *spage);
297static void scrub_wr_submit(struct scrub_ctx *sctx);
298static void scrub_wr_bio_end_io(struct bio *bio, int err);
299static void scrub_wr_bio_end_io_worker(struct btrfs_work *work);
300static int write_page_nocow(struct scrub_ctx *sctx,
301 u64 physical_for_dev_replace, struct page *page);
302static int copy_nocow_pages_for_inode(u64 inum, u64 offset, u64 root,
652f25a2 303 struct scrub_copy_nocow_ctx *ctx);
ff023aac
SB		 304static int copy_nocow_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
305 int mirror_num, u64 physical_for_dev_replace);
306static void copy_nocow_pages_worker(struct btrfs_work *work);
cb7ab021 307static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
3cb0929a 308static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
f55985f4 309static void scrub_put_ctx(struct scrub_ctx *sctx);
1623edeb
SB		 310
311
b6bfebc1
SB		 312static void scrub_pending_bio_inc(struct scrub_ctx *sctx)
313{
f55985f4 314 atomic_inc(&sctx->refs);
b6bfebc1
SB		 315 atomic_inc(&sctx->bios_in_flight);
316}
317
318static void scrub_pending_bio_dec(struct scrub_ctx *sctx)
319{
320 atomic_dec(&sctx->bios_in_flight);
321 wake_up(&sctx->list_wait);
f55985f4 322 scrub_put_ctx(sctx);
b6bfebc1
SB		 323}
324
cb7ab021 325static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
3cb0929a
WS		 326{
327 while (atomic_read(&fs_info->scrub_pause_req)) {
328 mutex_unlock(&fs_info->scrub_lock);
329 wait_event(fs_info->scrub_pause_wait,
330 atomic_read(&fs_info->scrub_pause_req) == 0);
331 mutex_lock(&fs_info->scrub_lock);
332 }
333}
334
0e22be89 335static void scrub_pause_on(struct btrfs_fs_info *fs_info)
cb7ab021
WS		 336{
337 atomic_inc(&fs_info->scrubs_paused);
338 wake_up(&fs_info->scrub_pause_wait);
0e22be89 339}
cb7ab021 340
0e22be89
Z		 341static void scrub_pause_off(struct btrfs_fs_info *fs_info)
342{
cb7ab021
WS		 343 mutex_lock(&fs_info->scrub_lock);
344 __scrub_blocked_if_needed(fs_info);
345 atomic_dec(&fs_info->scrubs_paused);
346 mutex_unlock(&fs_info->scrub_lock);
347
348 wake_up(&fs_info->scrub_pause_wait);
349}
350
0e22be89
Z		 351static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
352{
353 scrub_pause_on(fs_info);
354 scrub_pause_off(fs_info);
355}
356
b6bfebc1
SB		 357/*
358 * used for workers that require transaction commits (i.e., for the
359 * NOCOW case)
360 */
361static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx)
362{
363 struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
364
f55985f4 365 atomic_inc(&sctx->refs);
b6bfebc1
SB		 366 /*
367 * increment scrubs_running to prevent cancel requests from
368 * completing as long as a worker is running. we must also
369 * increment scrubs_paused to prevent deadlocking on pause
370 * requests used for transactions commits (as the worker uses a
371 * transaction context). it is safe to regard the worker
372 * as paused for all matters practical. effectively, we only
373 * avoid cancellation requests from completing.
374 */
375 mutex_lock(&fs_info->scrub_lock);
376 atomic_inc(&fs_info->scrubs_running);
377 atomic_inc(&fs_info->scrubs_paused);
378 mutex_unlock(&fs_info->scrub_lock);
32a44789
WS		 379
380 /*
381 * check if @scrubs_running=@scrubs_paused condition
382 * inside wait_event() is not an atomic operation.
383 * which means we may inc/dec @scrub_running/paused
384 * at any time. Let's wake up @scrub_pause_wait as
385 * much as we can to let commit transaction blocked less.
386 */
387 wake_up(&fs_info->scrub_pause_wait);
388
b6bfebc1
SB		 389 atomic_inc(&sctx->workers_pending);
390}
391
392/* used for workers that require transaction commits */
393static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx)
394{
395 struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
396
397 /*
398 * see scrub_pending_trans_workers_inc() why we're pretending
399 * to be paused in the scrub counters
400 */
401 mutex_lock(&fs_info->scrub_lock);
402 atomic_dec(&fs_info->scrubs_running);
403 atomic_dec(&fs_info->scrubs_paused);
404 mutex_unlock(&fs_info->scrub_lock);
405 atomic_dec(&sctx->workers_pending);
406 wake_up(&fs_info->scrub_pause_wait);
407 wake_up(&sctx->list_wait);
f55985f4 408 scrub_put_ctx(sctx);
b6bfebc1
SB		 409}
410
d9d181c1 411static void scrub_free_csums(struct scrub_ctx *sctx)
a2de733c 412{
d9d181c1 413 while (!list_empty(&sctx->csum_list)) {
a2de733c 414 struct btrfs_ordered_sum *sum;
d9d181c1 415 sum = list_first_entry(&sctx->csum_list,
a2de733c
AJ		 416 struct btrfs_ordered_sum, list);
417 list_del(&sum->list);
418 kfree(sum);
419 }
420}
421
d9d181c1 422static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx)
a2de733c
AJ		 423{
424 int i;
a2de733c 425
d9d181c1 426 if (!sctx)
a2de733c
AJ		 427 return;
428
ff023aac
SB		 429 scrub_free_wr_ctx(&sctx->wr_ctx);
430
b5d67f64 431 /* this can happen when scrub is cancelled */
d9d181c1
SB		 432 if (sctx->curr != -1) {
433 struct scrub_bio *sbio = sctx->bios[sctx->curr];
b5d67f64
SB		 434
435 for (i = 0; i < sbio->page_count; i++) {
ff023aac 436 WARN_ON(!sbio->pagev[i]->page);
b5d67f64
SB		 437 scrub_block_put(sbio->pagev[i]->sblock);
438 }
439 bio_put(sbio->bio);
440 }
441
ff023aac 442 for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
d9d181c1 443 struct scrub_bio *sbio = sctx->bios[i];
a2de733c
AJ		 444
445 if (!sbio)
446 break;
a2de733c
AJ		 447 kfree(sbio);
448 }
449
d9d181c1
SB		 450 scrub_free_csums(sctx);
451 kfree(sctx);
a2de733c
AJ		 452}
453
f55985f4
FM		 454static void scrub_put_ctx(struct scrub_ctx *sctx)
455{
456 if (atomic_dec_and_test(&sctx->refs))
457 scrub_free_ctx(sctx);
458}
459
a2de733c 460static noinline_for_stack
63a212ab 461struct scrub_ctx *scrub_setup_ctx(struct btrfs_device *dev, int is_dev_replace)
a2de733c 462{
d9d181c1 463 struct scrub_ctx *sctx;
a2de733c 464 int i;
a2de733c 465 struct btrfs_fs_info *fs_info = dev->dev_root->fs_info;
ff023aac
SB		 466 int pages_per_rd_bio;
467 int ret;
a2de733c 468
ff023aac
SB		 469 /*
470 * the setting of pages_per_rd_bio is correct for scrub but might
471 * be wrong for the dev_replace code where we might read from
472 * different devices in the initial huge bios. However, that
473 * code is able to correctly handle the case when adding a page
474 * to a bio fails.
475 */
476 if (dev->bdev)
477 pages_per_rd_bio = min_t(int, SCRUB_PAGES_PER_RD_BIO,
478 bio_get_nr_vecs(dev->bdev));
479 else
480 pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO;
d9d181c1
SB		 481 sctx = kzalloc(sizeof(*sctx), GFP_NOFS);
482 if (!sctx)
a2de733c 483 goto nomem;
f55985f4 484 atomic_set(&sctx->refs, 1);
63a212ab 485 sctx->is_dev_replace = is_dev_replace;
ff023aac 486 sctx->pages_per_rd_bio = pages_per_rd_bio;
d9d181c1 487 sctx->curr = -1;
a36cf8b8 488 sctx->dev_root = dev->dev_root;
ff023aac 489 for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
a2de733c
AJ		 490 struct scrub_bio *sbio;
491
492 sbio = kzalloc(sizeof(*sbio), GFP_NOFS);
493 if (!sbio)
494 goto nomem;
d9d181c1 495 sctx->bios[i] = sbio;
a2de733c 496
a2de733c 497 sbio->index = i;
d9d181c1 498 sbio->sctx = sctx;
b5d67f64 499 sbio->page_count = 0;
9e0af237
LB		 500 btrfs_init_work(&sbio->work, btrfs_scrub_helper,
501 scrub_bio_end_io_worker, NULL, NULL);
a2de733c 502
ff023aac 503 if (i != SCRUB_BIOS_PER_SCTX - 1)
d9d181c1 504 sctx->bios[i]->next_free = i + 1;
0ef8e451 505 else
d9d181c1
SB		 506 sctx->bios[i]->next_free = -1;
507 }
508 sctx->first_free = 0;
509 sctx->nodesize = dev->dev_root->nodesize;
d9d181c1 510 sctx->sectorsize = dev->dev_root->sectorsize;
b6bfebc1
SB		 511 atomic_set(&sctx->bios_in_flight, 0);
512 atomic_set(&sctx->workers_pending, 0);
d9d181c1
SB		 513 atomic_set(&sctx->cancel_req, 0);
514 sctx->csum_size = btrfs_super_csum_size(fs_info->super_copy);
515 INIT_LIST_HEAD(&sctx->csum_list);
516
517 spin_lock_init(&sctx->list_lock);
518 spin_lock_init(&sctx->stat_lock);
519 init_waitqueue_head(&sctx->list_wait);
ff023aac
SB		 520
521 ret = scrub_setup_wr_ctx(sctx, &sctx->wr_ctx, fs_info,
522 fs_info->dev_replace.tgtdev, is_dev_replace);
523 if (ret) {
524 scrub_free_ctx(sctx);
525 return ERR_PTR(ret);
526 }
d9d181c1 527 return sctx;
a2de733c
AJ		 528
529nomem:
d9d181c1 530 scrub_free_ctx(sctx);
a2de733c
AJ		 531 return ERR_PTR(-ENOMEM);
532}
533
ff023aac
SB		 534static int scrub_print_warning_inode(u64 inum, u64 offset, u64 root,
535 void *warn_ctx)
558540c1
JS		 536{
537 u64 isize;
538 u32 nlink;
539 int ret;
540 int i;
541 struct extent_buffer *eb;
542 struct btrfs_inode_item *inode_item;
ff023aac 543 struct scrub_warning *swarn = warn_ctx;
558540c1
JS		 544 struct btrfs_fs_info *fs_info = swarn->dev->dev_root->fs_info;
545 struct inode_fs_paths *ipath = NULL;
546 struct btrfs_root *local_root;
547 struct btrfs_key root_key;
1d4c08e0 548 struct btrfs_key key;
558540c1
JS		 549
550 root_key.objectid = root;
551 root_key.type = BTRFS_ROOT_ITEM_KEY;
552 root_key.offset = (u64)-1;
553 local_root = btrfs_read_fs_root_no_name(fs_info, &root_key);
554 if (IS_ERR(local_root)) {
555 ret = PTR_ERR(local_root);
556 goto err;
557 }
558
14692cc1
DS		 559 /*
560 * this makes the path point to (inum INODE_ITEM ioff)
561 */
1d4c08e0
DS		 562 key.objectid = inum;
563 key.type = BTRFS_INODE_ITEM_KEY;
564 key.offset = 0;
565
566 ret = btrfs_search_slot(NULL, local_root, &key, swarn->path, 0, 0);
558540c1
JS		 567 if (ret) {
568 btrfs_release_path(swarn->path);
569 goto err;
570 }
571
572 eb = swarn->path->nodes[0];
573 inode_item = btrfs_item_ptr(eb, swarn->path->slots[0],
574 struct btrfs_inode_item);
575 isize = btrfs_inode_size(eb, inode_item);
576 nlink = btrfs_inode_nlink(eb, inode_item);
577 btrfs_release_path(swarn->path);
578
579 ipath = init_ipath(4096, local_root, swarn->path);
26bdef54
DC		 580 if (IS_ERR(ipath)) {
581 ret = PTR_ERR(ipath);
582 ipath = NULL;
583 goto err;
584 }
558540c1
JS		 585 ret = paths_from_inode(inum, ipath);
586
587 if (ret < 0)
588 goto err;
589
590 /*
591 * we deliberately ignore the bit ipath might have been too small to
592 * hold all of the paths here
593 */
594 for (i = 0; i < ipath->fspath->elem_cnt; ++i)
efe120a0 595 printk_in_rcu(KERN_WARNING "BTRFS: %s at logical %llu on dev "
558540c1
JS		 596 "%s, sector %llu, root %llu, inode %llu, offset %llu, "
597 "length %llu, links %u (path: %s)\n", swarn->errstr,
606686ee 598 swarn->logical, rcu_str_deref(swarn->dev->name),
558540c1
JS		 599 (unsigned long long)swarn->sector, root, inum, offset,
600 min(isize - offset, (u64)PAGE_SIZE), nlink,
745c4d8e 601 (char *)(unsigned long)ipath->fspath->val[i]);
558540c1
JS		 602
603 free_ipath(ipath);
604 return 0;
605
606err:
efe120a0 607 printk_in_rcu(KERN_WARNING "BTRFS: %s at logical %llu on dev "
558540c1
JS		 608 "%s, sector %llu, root %llu, inode %llu, offset %llu: path "
609 "resolving failed with ret=%d\n", swarn->errstr,
606686ee 610 swarn->logical, rcu_str_deref(swarn->dev->name),
558540c1
JS		 611 (unsigned long long)swarn->sector, root, inum, offset, ret);
612
613 free_ipath(ipath);
614 return 0;
615}
616
b5d67f64 617static void scrub_print_warning(const char *errstr, struct scrub_block *sblock)
558540c1 618{
a36cf8b8
SB		 619 struct btrfs_device *dev;
620 struct btrfs_fs_info *fs_info;
558540c1
JS		 621 struct btrfs_path *path;
622 struct btrfs_key found_key;
623 struct extent_buffer *eb;
624 struct btrfs_extent_item *ei;
625 struct scrub_warning swarn;
69917e43
LB		 626 unsigned long ptr = 0;
627 u64 extent_item_pos;
628 u64 flags = 0;
558540c1 629 u64 ref_root;
69917e43 630 u32 item_size;
558540c1 631 u8 ref_level;
69917e43 632 int ret;
558540c1 633
a36cf8b8 634 WARN_ON(sblock->page_count < 1);
7a9e9987 635 dev = sblock->pagev[0]->dev;
a36cf8b8
SB		 636 fs_info = sblock->sctx->dev_root->fs_info;
637
558540c1 638 path = btrfs_alloc_path();
8b9456da
DS		 639 if (!path)
640 return;
558540c1 641
7a9e9987
SB		 642 swarn.sector = (sblock->pagev[0]->physical) >> 9;
643 swarn.logical = sblock->pagev[0]->logical;
558540c1 644 swarn.errstr = errstr;
a36cf8b8 645 swarn.dev = NULL;
558540c1 646
69917e43
LB		 647 ret = extent_from_logical(fs_info, swarn.logical, path, &found_key,
648 &flags);
558540c1
JS		 649 if (ret < 0)
650 goto out;
651
4692cf58 652 extent_item_pos = swarn.logical - found_key.objectid;
558540c1
JS		 653 swarn.extent_item_size = found_key.offset;
654
655 eb = path->nodes[0];
656 ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item);
657 item_size = btrfs_item_size_nr(eb, path->slots[0]);
658
69917e43 659 if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
558540c1 660 do {
6eda71d0
LB		 661 ret = tree_backref_for_extent(&ptr, eb, &found_key, ei,
662 item_size, &ref_root,
663 &ref_level);
606686ee 664 printk_in_rcu(KERN_WARNING
efe120a0 665 "BTRFS: %s at logical %llu on dev %s, "
558540c1 666 "sector %llu: metadata %s (level %d) in tree "
606686ee
JB		 667 "%llu\n", errstr, swarn.logical,
668 rcu_str_deref(dev->name),
558540c1
JS		 669 (unsigned long long)swarn.sector,
670 ref_level ? "node" : "leaf",
671 ret < 0 ? -1 : ref_level,
672 ret < 0 ? -1 : ref_root);
673 } while (ret != 1);
d8fe29e9 674 btrfs_release_path(path);
558540c1 675 } else {
d8fe29e9 676 btrfs_release_path(path);
558540c1 677 swarn.path = path;
a36cf8b8 678 swarn.dev = dev;
7a3ae2f8
JS		 679 iterate_extent_inodes(fs_info, found_key.objectid,
680 extent_item_pos, 1,
558540c1
JS		 681 scrub_print_warning_inode, &swarn);
682 }
683
684out:
685 btrfs_free_path(path);
558540c1
JS		 686}
687
ff023aac 688static int scrub_fixup_readpage(u64 inum, u64 offset, u64 root, void *fixup_ctx)
0ef8e451 689{
5da6fcbc 690 struct page *page = NULL;
0ef8e451 691 unsigned long index;
ff023aac 692 struct scrub_fixup_nodatasum *fixup = fixup_ctx;
0ef8e451 693 int ret;
5da6fcbc 694 int corrected = 0;
0ef8e451 695 struct btrfs_key key;
5da6fcbc 696 struct inode *inode = NULL;
6f1c3605 697 struct btrfs_fs_info *fs_info;
0ef8e451
JS		 698 u64 end = offset + PAGE_SIZE - 1;
699 struct btrfs_root *local_root;
6f1c3605 700 int srcu_index;
0ef8e451
JS		 701
702 key.objectid = root;
703 key.type = BTRFS_ROOT_ITEM_KEY;
704 key.offset = (u64)-1;
6f1c3605
LB		 705
706 fs_info = fixup->root->fs_info;
707 srcu_index = srcu_read_lock(&fs_info->subvol_srcu);
708
709 local_root = btrfs_read_fs_root_no_name(fs_info, &key);
710 if (IS_ERR(local_root)) {
711 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
0ef8e451 712 return PTR_ERR(local_root);
6f1c3605 713 }
0ef8e451
JS		 714
715 key.type = BTRFS_INODE_ITEM_KEY;
716 key.objectid = inum;
717 key.offset = 0;
6f1c3605
LB		 718 inode = btrfs_iget(fs_info->sb, &key, local_root, NULL);
719 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
0ef8e451
JS		 720 if (IS_ERR(inode))
721 return PTR_ERR(inode);
722
0ef8e451
JS		 723 index = offset >> PAGE_CACHE_SHIFT;
724
725 page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
5da6fcbc
JS		 726 if (!page) {
727 ret = -ENOMEM;
728 goto out;
729 }
730
731 if (PageUptodate(page)) {
5da6fcbc
JS		 732 if (PageDirty(page)) {
733 /*
734 * we need to write the data to the defect sector. the
735 * data that was in that sector is not in memory,
736 * because the page was modified. we must not write the
737 * modified page to that sector.
738 *
739 * TODO: what could be done here: wait for the delalloc
740 * runner to write out that page (might involve
741 * COW) and see whether the sector is still
742 * referenced afterwards.
743 *
744 * For the meantime, we'll treat this error
745 * incorrectable, although there is a chance that a
746 * later scrub will find the bad sector again and that
747 * there's no dirty page in memory, then.
748 */
749 ret = -EIO;
750 goto out;
751 }
1203b681 752 ret = repair_io_failure(inode, offset, PAGE_SIZE,
5da6fcbc 753 fixup->logical, page,
ffdd2018 754 offset - page_offset(page),
5da6fcbc
JS		 755 fixup->mirror_num);
756 unlock_page(page);
757 corrected = !ret;
758 } else {
759 /*
760 * we need to get good data first. the general readpage path
761 * will call repair_io_failure for us, we just have to make
762 * sure we read the bad mirror.
763 */
764 ret = set_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
765 EXTENT_DAMAGED, GFP_NOFS);
766 if (ret) {
767 /* set_extent_bits should give proper error */
768 WARN_ON(ret > 0);
769 if (ret > 0)
770 ret = -EFAULT;
771 goto out;
772 }
773
774 ret = extent_read_full_page(&BTRFS_I(inode)->io_tree, page,
775 btrfs_get_extent,
776 fixup->mirror_num);
777 wait_on_page_locked(page);
778
779 corrected = !test_range_bit(&BTRFS_I(inode)->io_tree, offset,
780 end, EXTENT_DAMAGED, 0, NULL);
781 if (!corrected)
782 clear_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
783 EXTENT_DAMAGED, GFP_NOFS);
784 }
785
786out:
787 if (page)
788 put_page(page);
7fb18a06
TK		 789
790 iput(inode);
0ef8e451
JS		 791
792 if (ret < 0)
793 return ret;
794
795 if (ret == 0 && corrected) {
796 /*
797 * we only need to call readpage for one of the inodes belonging
798 * to this extent. so make iterate_extent_inodes stop
799 */
800 return 1;
801 }
802
803 return -EIO;
804}
805
806static void scrub_fixup_nodatasum(struct btrfs_work *work)
807{
808 int ret;
809 struct scrub_fixup_nodatasum *fixup;
d9d181c1 810 struct scrub_ctx *sctx;
0ef8e451 811 struct btrfs_trans_handle *trans = NULL;
0ef8e451
JS		 812 struct btrfs_path *path;
813 int uncorrectable = 0;
814
815 fixup = container_of(work, struct scrub_fixup_nodatasum, work);
d9d181c1 816 sctx = fixup->sctx;
0ef8e451
JS		 817
818 path = btrfs_alloc_path();
819 if (!path) {
d9d181c1
SB		 820 spin_lock(&sctx->stat_lock);
821 ++sctx->stat.malloc_errors;
822 spin_unlock(&sctx->stat_lock);
0ef8e451
JS		 823 uncorrectable = 1;
824 goto out;
825 }
826
827 trans = btrfs_join_transaction(fixup->root);
828 if (IS_ERR(trans)) {
829 uncorrectable = 1;
830 goto out;
831 }
832
833 /*
834 * the idea is to trigger a regular read through the standard path. we
835 * read a page from the (failed) logical address by specifying the
836 * corresponding copynum of the failed sector. thus, that readpage is
837 * expected to fail.
838 * that is the point where on-the-fly error correction will kick in
839 * (once it's finished) and rewrite the failed sector if a good copy
840 * can be found.
841 */
842 ret = iterate_inodes_from_logical(fixup->logical, fixup->root->fs_info,
843 path, scrub_fixup_readpage,
844 fixup);
845 if (ret < 0) {
846 uncorrectable = 1;
847 goto out;
848 }
849 WARN_ON(ret != 1);
850
d9d181c1
SB		 851 spin_lock(&sctx->stat_lock);
852 ++sctx->stat.corrected_errors;
853 spin_unlock(&sctx->stat_lock);
0ef8e451
JS		 854
855out:
856 if (trans && !IS_ERR(trans))
857 btrfs_end_transaction(trans, fixup->root);
858 if (uncorrectable) {
d9d181c1
SB		 859 spin_lock(&sctx->stat_lock);
860 ++sctx->stat.uncorrectable_errors;
861 spin_unlock(&sctx->stat_lock);
ff023aac
SB		 862 btrfs_dev_replace_stats_inc(
863 &sctx->dev_root->fs_info->dev_replace.
864 num_uncorrectable_read_errors);
efe120a0
FH		 865 printk_ratelimited_in_rcu(KERN_ERR "BTRFS: "
866 "unable to fixup (nodatasum) error at logical %llu on dev %s\n",
c1c9ff7c 867 fixup->logical, rcu_str_deref(fixup->dev->name));
0ef8e451
JS		 868 }
869
870 btrfs_free_path(path);
871 kfree(fixup);
872
b6bfebc1 873 scrub_pending_trans_workers_dec(sctx);
0ef8e451
JS		 874}
875
af8e2d1d
MX		 876static inline void scrub_get_recover(struct scrub_recover *recover)
877{
878 atomic_inc(&recover->refs);
879}
880
881static inline void scrub_put_recover(struct scrub_recover *recover)
882{
883 if (atomic_dec_and_test(&recover->refs)) {
6e9606d2 884 btrfs_put_bbio(recover->bbio);
af8e2d1d
MX		 885 kfree(recover);
886 }
887}
888
a2de733c 889/*
b5d67f64
SB		 890 * scrub_handle_errored_block gets called when either verification of the
891 * pages failed or the bio failed to read, e.g. with EIO. In the latter
892 * case, this function handles all pages in the bio, even though only one
893 * may be bad.
894 * The goal of this function is to repair the errored block by using the
895 * contents of one of the mirrors.
a2de733c 896 */
b5d67f64 897static int scrub_handle_errored_block(struct scrub_block *sblock_to_check)
a2de733c 898{
d9d181c1 899 struct scrub_ctx *sctx = sblock_to_check->sctx;
a36cf8b8 900 struct btrfs_device *dev;
b5d67f64
SB		 901 struct btrfs_fs_info *fs_info;
902 u64 length;
903 u64 logical;
904 u64 generation;
905 unsigned int failed_mirror_index;
906 unsigned int is_metadata;
907 unsigned int have_csum;
908 u8 *csum;
909 struct scrub_block *sblocks_for_recheck; /* holds one for each mirror */
910 struct scrub_block *sblock_bad;
911 int ret;
912 int mirror_index;
913 int page_num;
914 int success;
558540c1 915 static DEFINE_RATELIMIT_STATE(_rs, DEFAULT_RATELIMIT_INTERVAL,
b5d67f64
SB		 916 DEFAULT_RATELIMIT_BURST);
917
918 BUG_ON(sblock_to_check->page_count < 1);
a36cf8b8 919 fs_info = sctx->dev_root->fs_info;
4ded4f63
SB		 920 if (sblock_to_check->pagev[0]->flags & BTRFS_EXTENT_FLAG_SUPER) {
921 /*
922 * if we find an error in a super block, we just report it.
923 * They will get written with the next transaction commit
924 * anyway
925 */
926 spin_lock(&sctx->stat_lock);
927 ++sctx->stat.super_errors;
928 spin_unlock(&sctx->stat_lock);
929 return 0;
930 }
b5d67f64 931 length = sblock_to_check->page_count * PAGE_SIZE;
7a9e9987
SB		 932 logical = sblock_to_check->pagev[0]->logical;
933 generation = sblock_to_check->pagev[0]->generation;
934 BUG_ON(sblock_to_check->pagev[0]->mirror_num < 1);
935 failed_mirror_index = sblock_to_check->pagev[0]->mirror_num - 1;
936 is_metadata = !(sblock_to_check->pagev[0]->flags &
b5d67f64 937 BTRFS_EXTENT_FLAG_DATA);
7a9e9987
SB		 938 have_csum = sblock_to_check->pagev[0]->have_csum;
939 csum = sblock_to_check->pagev[0]->csum;
940 dev = sblock_to_check->pagev[0]->dev;
13db62b7 941
ff023aac
SB		 942 if (sctx->is_dev_replace && !is_metadata && !have_csum) {
943 sblocks_for_recheck = NULL;
944 goto nodatasum_case;
945 }
946
b5d67f64
SB		 947 /*
948 * read all mirrors one after the other. This includes to
949 * re-read the extent or metadata block that failed (that was
950 * the cause that this fixup code is called) another time,
951 * page by page this time in order to know which pages
952 * caused I/O errors and which ones are good (for all mirrors).
953 * It is the goal to handle the situation when more than one
954 * mirror contains I/O errors, but the errors do not
955 * overlap, i.e. the data can be repaired by selecting the
956 * pages from those mirrors without I/O error on the
957 * particular pages. One example (with blocks >= 2 * PAGE_SIZE)
958 * would be that mirror #1 has an I/O error on the first page,
959 * the second page is good, and mirror #2 has an I/O error on
960 * the second page, but the first page is good.
961 * Then the first page of the first mirror can be repaired by
962 * taking the first page of the second mirror, and the
963 * second page of the second mirror can be repaired by
964 * copying the contents of the 2nd page of the 1st mirror.
965 * One more note: if the pages of one mirror contain I/O
966 * errors, the checksum cannot be verified. In order to get
967 * the best data for repairing, the first attempt is to find
968 * a mirror without I/O errors and with a validated checksum.
969 * Only if this is not possible, the pages are picked from
970 * mirrors with I/O errors without considering the checksum.
971 * If the latter is the case, at the end, the checksum of the
972 * repaired area is verified in order to correctly maintain
973 * the statistics.
974 */
975
31e818fe
DS		 976 sblocks_for_recheck = kcalloc(BTRFS_MAX_MIRRORS,
977 sizeof(*sblocks_for_recheck), GFP_NOFS);
b5d67f64 978 if (!sblocks_for_recheck) {
d9d181c1
SB		 979 spin_lock(&sctx->stat_lock);
980 sctx->stat.malloc_errors++;
981 sctx->stat.read_errors++;
982 sctx->stat.uncorrectable_errors++;
983 spin_unlock(&sctx->stat_lock);
a36cf8b8 984 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
b5d67f64 985 goto out;
a2de733c
AJ		 986 }
987
b5d67f64 988 /* setup the context, map the logical blocks and alloc the pages */
be50a8dd 989 ret = scrub_setup_recheck_block(sblock_to_check, sblocks_for_recheck);
b5d67f64 990 if (ret) {
d9d181c1
SB		 991 spin_lock(&sctx->stat_lock);
992 sctx->stat.read_errors++;
993 sctx->stat.uncorrectable_errors++;
994 spin_unlock(&sctx->stat_lock);
a36cf8b8 995 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
b5d67f64
SB		 996 goto out;
997 }
998 BUG_ON(failed_mirror_index >= BTRFS_MAX_MIRRORS);
999 sblock_bad = sblocks_for_recheck + failed_mirror_index;
13db62b7 1000
b5d67f64 1001 /* build and submit the bios for the failed mirror, check checksums */
34f5c8e9 1002 scrub_recheck_block(fs_info, sblock_bad, is_metadata, have_csum,
af8e2d1d 1003 csum, generation, sctx->csum_size, 1);
a2de733c 1004
b5d67f64
SB		 1005 if (!sblock_bad->header_error && !sblock_bad->checksum_error &&
1006 sblock_bad->no_io_error_seen) {
1007 /*
1008 * the error disappeared after reading page by page, or
1009 * the area was part of a huge bio and other parts of the
1010 * bio caused I/O errors, or the block layer merged several
1011 * read requests into one and the error is caused by a
1012 * different bio (usually one of the two latter cases is
1013 * the cause)
1014 */
d9d181c1
SB		 1015 spin_lock(&sctx->stat_lock);
1016 sctx->stat.unverified_errors++;
5a6ac9ea 1017 sblock_to_check->data_corrected = 1;
d9d181c1 1018 spin_unlock(&sctx->stat_lock);
a2de733c 1019
ff023aac
SB		 1020 if (sctx->is_dev_replace)
1021 scrub_write_block_to_dev_replace(sblock_bad);
b5d67f64 1022 goto out;
a2de733c 1023 }
a2de733c 1024
b5d67f64 1025 if (!sblock_bad->no_io_error_seen) {
d9d181c1
SB		 1026 spin_lock(&sctx->stat_lock);
1027 sctx->stat.read_errors++;
1028 spin_unlock(&sctx->stat_lock);
b5d67f64
SB		 1029 if (__ratelimit(&_rs))
1030 scrub_print_warning("i/o error", sblock_to_check);
a36cf8b8 1031 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
b5d67f64 1032 } else if (sblock_bad->checksum_error) {
d9d181c1
SB		 1033 spin_lock(&sctx->stat_lock);
1034 sctx->stat.csum_errors++;
1035 spin_unlock(&sctx->stat_lock);
b5d67f64
SB		 1036 if (__ratelimit(&_rs))
1037 scrub_print_warning("checksum error", sblock_to_check);
a36cf8b8 1038 btrfs_dev_stat_inc_and_print(dev,
442a4f63 1039 BTRFS_DEV_STAT_CORRUPTION_ERRS);
b5d67f64 1040 } else if (sblock_bad->header_error) {
d9d181c1
SB		 1041 spin_lock(&sctx->stat_lock);
1042 sctx->stat.verify_errors++;
1043 spin_unlock(&sctx->stat_lock);
b5d67f64
SB		 1044 if (__ratelimit(&_rs))
1045 scrub_print_warning("checksum/header error",
1046 sblock_to_check);
442a4f63 1047 if (sblock_bad->generation_error)
a36cf8b8 1048 btrfs_dev_stat_inc_and_print(dev,
442a4f63
SB		 1049 BTRFS_DEV_STAT_GENERATION_ERRS);
1050 else
a36cf8b8 1051 btrfs_dev_stat_inc_and_print(dev,
442a4f63 1052 BTRFS_DEV_STAT_CORRUPTION_ERRS);
b5d67f64 1053 }
a2de733c 1054
33ef30ad
ID		 1055 if (sctx->readonly) {
1056 ASSERT(!sctx->is_dev_replace);
1057 goto out;
1058 }
a2de733c 1059
b5d67f64
SB		 1060 if (!is_metadata && !have_csum) {
1061 struct scrub_fixup_nodatasum *fixup_nodatasum;
a2de733c 1062
ff023aac
SB		 1063 WARN_ON(sctx->is_dev_replace);
1064
b25c94c5
ZL		 1065nodatasum_case:
1066
b5d67f64
SB		 1067 /*
1068 * !is_metadata and !have_csum, this means that the data
1069 * might not be COW'ed, that it might be modified
1070 * concurrently. The general strategy to work on the
1071 * commit root does not help in the case when COW is not
1072 * used.
1073 */
1074 fixup_nodatasum = kzalloc(sizeof(*fixup_nodatasum), GFP_NOFS);
1075 if (!fixup_nodatasum)
1076 goto did_not_correct_error;
d9d181c1 1077 fixup_nodatasum->sctx = sctx;
a36cf8b8 1078 fixup_nodatasum->dev = dev;
b5d67f64
SB		 1079 fixup_nodatasum->logical = logical;
1080 fixup_nodatasum->root = fs_info->extent_root;
1081 fixup_nodatasum->mirror_num = failed_mirror_index + 1;
b6bfebc1 1082 scrub_pending_trans_workers_inc(sctx);
9e0af237
LB		 1083 btrfs_init_work(&fixup_nodatasum->work, btrfs_scrub_helper,
1084 scrub_fixup_nodatasum, NULL, NULL);
0339ef2f
QW		 1085 btrfs_queue_work(fs_info->scrub_workers,
1086 &fixup_nodatasum->work);
b5d67f64 1087 goto out;
a2de733c
AJ		 1088 }
1089
b5d67f64
SB		 1090 /*
1091 * now build and submit the bios for the other mirrors, check
cb2ced73
SB		 1092 * checksums.
1093 * First try to pick the mirror which is completely without I/O
b5d67f64
SB		 1094 * errors and also does not have a checksum error.
1095 * If one is found, and if a checksum is present, the full block
1096 * that is known to contain an error is rewritten. Afterwards
1097 * the block is known to be corrected.
1098 * If a mirror is found which is completely correct, and no
1099 * checksum is present, only those pages are rewritten that had
1100 * an I/O error in the block to be repaired, since it cannot be
1101 * determined, which copy of the other pages is better (and it
1102 * could happen otherwise that a correct page would be
1103 * overwritten by a bad one).
1104 */
1105 for (mirror_index = 0;
1106 mirror_index < BTRFS_MAX_MIRRORS &&
1107 sblocks_for_recheck[mirror_index].page_count > 0;
1108 mirror_index++) {
cb2ced73 1109 struct scrub_block *sblock_other;
b5d67f64 1110
cb2ced73
SB		 1111 if (mirror_index == failed_mirror_index)
1112 continue;
1113 sblock_other = sblocks_for_recheck + mirror_index;
1114
1115 /* build and submit the bios, check checksums */
34f5c8e9
SB		 1116 scrub_recheck_block(fs_info, sblock_other, is_metadata,
1117 have_csum, csum, generation,
af8e2d1d 1118 sctx->csum_size, 0);
34f5c8e9
SB		 1119
1120 if (!sblock_other->header_error &&
b5d67f64
SB		 1121 !sblock_other->checksum_error &&
1122 sblock_other->no_io_error_seen) {
ff023aac
SB		 1123 if (sctx->is_dev_replace) {
1124 scrub_write_block_to_dev_replace(sblock_other);
114ab50d 1125 goto corrected_error;
ff023aac 1126 } else {
ff023aac 1127 ret = scrub_repair_block_from_good_copy(
114ab50d
ZL		 1128 sblock_bad, sblock_other);
1129 if (!ret)
1130 goto corrected_error;
ff023aac 1131 }
b5d67f64
SB		 1132 }
1133 }
a2de733c 1134
b968fed1
ZL		 1135 if (sblock_bad->no_io_error_seen && !sctx->is_dev_replace)
1136 goto did_not_correct_error;
ff023aac
SB		 1137
1138 /*
ff023aac 1139 * In case of I/O errors in the area that is supposed to be
b5d67f64
SB		 1140 * repaired, continue by picking good copies of those pages.
1141 * Select the good pages from mirrors to rewrite bad pages from
1142 * the area to fix. Afterwards verify the checksum of the block
1143 * that is supposed to be repaired. This verification step is
1144 * only done for the purpose of statistic counting and for the
1145 * final scrub report, whether errors remain.
1146 * A perfect algorithm could make use of the checksum and try
1147 * all possible combinations of pages from the different mirrors
1148 * until the checksum verification succeeds. For example, when
1149 * the 2nd page of mirror #1 faces I/O errors, and the 2nd page
1150 * of mirror #2 is readable but the final checksum test fails,
1151 * then the 2nd page of mirror #3 could be tried, whether now
1152 * the final checksum succeedes. But this would be a rare
1153 * exception and is therefore not implemented. At least it is
1154 * avoided that the good copy is overwritten.
1155 * A more useful improvement would be to pick the sectors
1156 * without I/O error based on sector sizes (512 bytes on legacy
1157 * disks) instead of on PAGE_SIZE. Then maybe 512 byte of one
1158 * mirror could be repaired by taking 512 byte of a different
1159 * mirror, even if other 512 byte sectors in the same PAGE_SIZE
1160 * area are unreadable.
a2de733c 1161 */
b5d67f64 1162 success = 1;
b968fed1
ZL		 1163 for (page_num = 0; page_num < sblock_bad->page_count;
1164 page_num++) {
7a9e9987 1165 struct scrub_page *page_bad = sblock_bad->pagev[page_num];
b968fed1 1166 struct scrub_block *sblock_other = NULL;
b5d67f64 1167
b968fed1
ZL		 1168 /* skip no-io-error page in scrub */
1169 if (!page_bad->io_error && !sctx->is_dev_replace)
a2de733c 1170 continue;
b5d67f64 1171
b968fed1
ZL		 1172 /* try to find no-io-error page in mirrors */
1173 if (page_bad->io_error) {
1174 for (mirror_index = 0;
1175 mirror_index < BTRFS_MAX_MIRRORS &&
1176 sblocks_for_recheck[mirror_index].page_count > 0;
1177 mirror_index++) {
1178 if (!sblocks_for_recheck[mirror_index].
1179 pagev[page_num]->io_error) {
1180 sblock_other = sblocks_for_recheck +
1181 mirror_index;
1182 break;
b5d67f64
SB		 1183 }
1184 }
b968fed1
ZL		 1185 if (!sblock_other)
1186 success = 0;
96e36920 1187 }
a2de733c 1188
b968fed1
ZL		 1189 if (sctx->is_dev_replace) {
1190 /*
1191 * did not find a mirror to fetch the page
1192 * from. scrub_write_page_to_dev_replace()
1193 * handles this case (page->io_error), by
1194 * filling the block with zeros before
1195 * submitting the write request
1196 */
1197 if (!sblock_other)
1198 sblock_other = sblock_bad;
1199
1200 if (scrub_write_page_to_dev_replace(sblock_other,
1201 page_num) != 0) {
1202 btrfs_dev_replace_stats_inc(
1203 &sctx->dev_root->
1204 fs_info->dev_replace.
1205 num_write_errors);
1206 success = 0;
1207 }
1208 } else if (sblock_other) {
1209 ret = scrub_repair_page_from_good_copy(sblock_bad,
1210 sblock_other,
1211 page_num, 0);
1212 if (0 == ret)
1213 page_bad->io_error = 0;
1214 else
1215 success = 0;
b5d67f64 1216 }
a2de733c 1217 }
a2de733c 1218
b968fed1 1219 if (success && !sctx->is_dev_replace) {
b5d67f64
SB		 1220 if (is_metadata || have_csum) {
1221 /*
1222 * need to verify the checksum now that all
1223 * sectors on disk are repaired (the write
1224 * request for data to be repaired is on its way).
1225 * Just be lazy and use scrub_recheck_block()
1226 * which re-reads the data before the checksum
1227 * is verified, but most likely the data comes out
1228 * of the page cache.
1229 */
34f5c8e9
SB		 1230 scrub_recheck_block(fs_info, sblock_bad,
1231 is_metadata, have_csum, csum,
af8e2d1d 1232 generation, sctx->csum_size, 1);
34f5c8e9 1233 if (!sblock_bad->header_error &&
b5d67f64
SB		 1234 !sblock_bad->checksum_error &&
1235 sblock_bad->no_io_error_seen)
1236 goto corrected_error;
1237 else
1238 goto did_not_correct_error;
1239 } else {
1240corrected_error:
d9d181c1
SB		 1241 spin_lock(&sctx->stat_lock);
1242 sctx->stat.corrected_errors++;
5a6ac9ea 1243 sblock_to_check->data_corrected = 1;
d9d181c1 1244 spin_unlock(&sctx->stat_lock);
606686ee 1245 printk_ratelimited_in_rcu(KERN_ERR
efe120a0 1246 "BTRFS: fixed up error at logical %llu on dev %s\n",
c1c9ff7c 1247 logical, rcu_str_deref(dev->name));
8628764e 1248 }
b5d67f64
SB		 1249 } else {
1250did_not_correct_error:
d9d181c1
SB		 1251 spin_lock(&sctx->stat_lock);
1252 sctx->stat.uncorrectable_errors++;
1253 spin_unlock(&sctx->stat_lock);
606686ee 1254 printk_ratelimited_in_rcu(KERN_ERR
efe120a0 1255 "BTRFS: unable to fixup (regular) error at logical %llu on dev %s\n",
c1c9ff7c 1256 logical, rcu_str_deref(dev->name));
96e36920 1257 }
a2de733c 1258
b5d67f64
SB		 1259out:
1260 if (sblocks_for_recheck) {
1261 for (mirror_index = 0; mirror_index < BTRFS_MAX_MIRRORS;
1262 mirror_index++) {
1263 struct scrub_block *sblock = sblocks_for_recheck +
1264 mirror_index;
af8e2d1d 1265 struct scrub_recover *recover;
b5d67f64
SB		 1266 int page_index;
1267
7a9e9987
SB		 1268 for (page_index = 0; page_index < sblock->page_count;
1269 page_index++) {
1270 sblock->pagev[page_index]->sblock = NULL;
af8e2d1d
MX		 1271 recover = sblock->pagev[page_index]->recover;
1272 if (recover) {
1273 scrub_put_recover(recover);
1274 sblock->pagev[page_index]->recover =
1275 NULL;
1276 }
7a9e9987
SB		 1277 scrub_page_put(sblock->pagev[page_index]);
1278 }
b5d67f64
SB		 1279 }
1280 kfree(sblocks_for_recheck);
1281 }
a2de733c 1282
b5d67f64
SB		 1283 return 0;
1284}
a2de733c 1285
8e5cfb55 1286static inline int scrub_nr_raid_mirrors(struct btrfs_bio *bbio)
af8e2d1d 1287{
10f11900
ZL		 1288 if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID5)
1289 return 2;
1290 else if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID6)
1291 return 3;
1292 else
af8e2d1d 1293 return (int)bbio->num_stripes;
af8e2d1d
MX		 1294}
1295
10f11900
ZL		 1296static inline void scrub_stripe_index_and_offset(u64 logical, u64 map_type,
1297 u64 *raid_map,
af8e2d1d
MX		 1298 u64 mapped_length,
1299 int nstripes, int mirror,
1300 int *stripe_index,
1301 u64 *stripe_offset)
1302{
1303 int i;
1304
ffe2d203 1305 if (map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
af8e2d1d
MX		 1306 /* RAID5/6 */
1307 for (i = 0; i < nstripes; i++) {
1308 if (raid_map[i] == RAID6_Q_STRIPE ||
1309 raid_map[i] == RAID5_P_STRIPE)
1310 continue;
1311
1312 if (logical >= raid_map[i] &&
1313 logical < raid_map[i] + mapped_length)
1314 break;
1315 }
1316
1317 *stripe_index = i;
1318 *stripe_offset = logical - raid_map[i];
1319 } else {
1320 /* The other RAID type */
1321 *stripe_index = mirror;
1322 *stripe_offset = 0;
1323 }
1324}
1325
be50a8dd 1326static int scrub_setup_recheck_block(struct scrub_block *original_sblock,
b5d67f64
SB		 1327 struct scrub_block *sblocks_for_recheck)
1328{
be50a8dd
ZL		 1329 struct scrub_ctx *sctx = original_sblock->sctx;
1330 struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
1331 u64 length = original_sblock->page_count * PAGE_SIZE;
1332 u64 logical = original_sblock->pagev[0]->logical;
af8e2d1d
MX		 1333 struct scrub_recover *recover;
1334 struct btrfs_bio *bbio;
af8e2d1d
MX		 1335 u64 sublen;
1336 u64 mapped_length;
1337 u64 stripe_offset;
1338 int stripe_index;
be50a8dd 1339 int page_index = 0;
b5d67f64 1340 int mirror_index;
af8e2d1d 1341 int nmirrors;
b5d67f64
SB		 1342 int ret;
1343
1344 /*
57019345 1345 * note: the two members refs and outstanding_pages
b5d67f64
SB		 1346 * are not used (and not set) in the blocks that are used for
1347 * the recheck procedure
1348 */
1349
b5d67f64 1350 while (length > 0) {
af8e2d1d
MX		 1351 sublen = min_t(u64, length, PAGE_SIZE);
1352 mapped_length = sublen;
1353 bbio = NULL;
a2de733c 1354
b5d67f64
SB		 1355 /*
1356 * with a length of PAGE_SIZE, each returned stripe
1357 * represents one mirror
1358 */
af8e2d1d 1359 ret = btrfs_map_sblock(fs_info, REQ_GET_READ_MIRRORS, logical,
8e5cfb55 1360 &mapped_length, &bbio, 0, 1);
b5d67f64 1361 if (ret || !bbio || mapped_length < sublen) {
6e9606d2 1362 btrfs_put_bbio(bbio);
b5d67f64
SB		 1363 return -EIO;
1364 }
a2de733c 1365
af8e2d1d
MX		 1366 recover = kzalloc(sizeof(struct scrub_recover), GFP_NOFS);
1367 if (!recover) {
6e9606d2 1368 btrfs_put_bbio(bbio);
af8e2d1d
MX		 1369 return -ENOMEM;
1370 }
1371
1372 atomic_set(&recover->refs, 1);
1373 recover->bbio = bbio;
af8e2d1d
MX		 1374 recover->map_length = mapped_length;
1375
ff023aac 1376 BUG_ON(page_index >= SCRUB_PAGES_PER_RD_BIO);
af8e2d1d 1377
be50a8dd 1378 nmirrors = min(scrub_nr_raid_mirrors(bbio), BTRFS_MAX_MIRRORS);
10f11900 1379
af8e2d1d 1380 for (mirror_index = 0; mirror_index < nmirrors;
b5d67f64
SB		 1381 mirror_index++) {
1382 struct scrub_block *sblock;
1383 struct scrub_page *page;
1384
b5d67f64 1385 sblock = sblocks_for_recheck + mirror_index;
7a9e9987
SB		 1386 sblock->sctx = sctx;
1387 page = kzalloc(sizeof(*page), GFP_NOFS);
1388 if (!page) {
1389leave_nomem:
d9d181c1
SB		 1390 spin_lock(&sctx->stat_lock);
1391 sctx->stat.malloc_errors++;
1392 spin_unlock(&sctx->stat_lock);
af8e2d1d 1393 scrub_put_recover(recover);
b5d67f64
SB		 1394 return -ENOMEM;
1395 }
7a9e9987
SB		 1396 scrub_page_get(page);
1397 sblock->pagev[page_index] = page;
1398 page->logical = logical;
af8e2d1d 1399
10f11900
ZL		 1400 scrub_stripe_index_and_offset(logical,
1401 bbio->map_type,
1402 bbio->raid_map,
af8e2d1d 1403 mapped_length,
e34c330d
ZL		 1404 bbio->num_stripes -
1405 bbio->num_tgtdevs,
af8e2d1d
MX		 1406 mirror_index,
1407 &stripe_index,
1408 &stripe_offset);
1409 page->physical = bbio->stripes[stripe_index].physical +
1410 stripe_offset;
1411 page->dev = bbio->stripes[stripe_index].dev;
1412
ff023aac
SB		 1413 BUG_ON(page_index >= original_sblock->page_count);
1414 page->physical_for_dev_replace =
1415 original_sblock->pagev[page_index]->
1416 physical_for_dev_replace;
7a9e9987 1417 /* for missing devices, dev->bdev is NULL */
7a9e9987 1418 page->mirror_num = mirror_index + 1;
b5d67f64 1419 sblock->page_count++;
7a9e9987
SB		 1420 page->page = alloc_page(GFP_NOFS);
1421 if (!page->page)
1422 goto leave_nomem;
af8e2d1d
MX		 1423
1424 scrub_get_recover(recover);
1425 page->recover = recover;
b5d67f64 1426 }
af8e2d1d 1427 scrub_put_recover(recover);
b5d67f64
SB		 1428 length -= sublen;
1429 logical += sublen;
1430 page_index++;
1431 }
1432
1433 return 0;
96e36920
ID		 1434}
1435
af8e2d1d
MX		 1436struct scrub_bio_ret {
1437 struct completion event;
1438 int error;
1439};
1440
1441static void scrub_bio_wait_endio(struct bio *bio, int error)
1442{
1443 struct scrub_bio_ret *ret = bio->bi_private;
1444
1445 ret->error = error;
1446 complete(&ret->event);
1447}
1448
1449static inline int scrub_is_page_on_raid56(struct scrub_page *page)
1450{
10f11900 1451 return page->recover &&
ffe2d203 1452 (page->recover->bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK);
af8e2d1d
MX		 1453}
1454
1455static int scrub_submit_raid56_bio_wait(struct btrfs_fs_info *fs_info,
1456 struct bio *bio,
1457 struct scrub_page *page)
1458{
1459 struct scrub_bio_ret done;
1460 int ret;
1461
1462 init_completion(&done.event);
1463 done.error = 0;
1464 bio->bi_iter.bi_sector = page->logical >> 9;
1465 bio->bi_private = &done;
1466 bio->bi_end_io = scrub_bio_wait_endio;
1467
1468 ret = raid56_parity_recover(fs_info->fs_root, bio, page->recover->bbio,
af8e2d1d 1469 page->recover->map_length,
4245215d 1470 page->mirror_num, 0);
af8e2d1d
MX		 1471 if (ret)
1472 return ret;
1473
1474 wait_for_completion(&done.event);
1475 if (done.error)
1476 return -EIO;
1477
1478 return 0;
1479}
1480
b5d67f64
SB		 1481/*
1482 * this function will check the on disk data for checksum errors, header
1483 * errors and read I/O errors. If any I/O errors happen, the exact pages
1484 * which are errored are marked as being bad. The goal is to enable scrub
1485 * to take those pages that are not errored from all the mirrors so that
1486 * the pages that are errored in the just handled mirror can be repaired.
1487 */
34f5c8e9
SB		 1488static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
1489 struct scrub_block *sblock, int is_metadata,
1490 int have_csum, u8 *csum, u64 generation,
af8e2d1d 1491 u16 csum_size, int retry_failed_mirror)
96e36920 1492{
b5d67f64 1493 int page_num;
96e36920 1494
b5d67f64
SB		 1495 sblock->no_io_error_seen = 1;
1496 sblock->header_error = 0;
1497 sblock->checksum_error = 0;
96e36920 1498
b5d67f64
SB		 1499 for (page_num = 0; page_num < sblock->page_count; page_num++) {
1500 struct bio *bio;
7a9e9987 1501 struct scrub_page *page = sblock->pagev[page_num];
b5d67f64 1502
442a4f63 1503 if (page->dev->bdev == NULL) {
ea9947b4
SB		 1504 page->io_error = 1;
1505 sblock->no_io_error_seen = 0;
1506 continue;
1507 }
1508
7a9e9987 1509 WARN_ON(!page->page);
9be3395b 1510 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
34f5c8e9
SB		 1511 if (!bio) {
1512 page->io_error = 1;
1513 sblock->no_io_error_seen = 0;
1514 continue;
1515 }
442a4f63 1516 bio->bi_bdev = page->dev->bdev;
b5d67f64 1517
34f5c8e9 1518 bio_add_page(bio, page->page, PAGE_SIZE, 0);
af8e2d1d
MX		 1519 if (!retry_failed_mirror && scrub_is_page_on_raid56(page)) {
1520 if (scrub_submit_raid56_bio_wait(fs_info, bio, page))
1521 sblock->no_io_error_seen = 0;
1522 } else {
1523 bio->bi_iter.bi_sector = page->physical >> 9;
1524
1525 if (btrfsic_submit_bio_wait(READ, bio))
1526 sblock->no_io_error_seen = 0;
1527 }
33879d45 1528
b5d67f64
SB		 1529 bio_put(bio);
1530 }
96e36920 1531
b5d67f64
SB		 1532 if (sblock->no_io_error_seen)
1533 scrub_recheck_block_checksum(fs_info, sblock, is_metadata,
1534 have_csum, csum, generation,
1535 csum_size);
1536
34f5c8e9 1537 return;
a2de733c
AJ		 1538}
1539
17a9be2f
MX		 1540static inline int scrub_check_fsid(u8 fsid[],
1541 struct scrub_page *spage)
1542{
1543 struct btrfs_fs_devices *fs_devices = spage->dev->fs_devices;
1544 int ret;
1545
1546 ret = memcmp(fsid, fs_devices->fsid, BTRFS_UUID_SIZE);
1547 return !ret;
1548}
1549
b5d67f64
SB		 1550static void scrub_recheck_block_checksum(struct btrfs_fs_info *fs_info,
1551 struct scrub_block *sblock,
1552 int is_metadata, int have_csum,
1553 const u8 *csum, u64 generation,
1554 u16 csum_size)
a2de733c 1555{
b5d67f64
SB		 1556 int page_num;
1557 u8 calculated_csum[BTRFS_CSUM_SIZE];
1558 u32 crc = ~(u32)0;
b5d67f64
SB		 1559 void *mapped_buffer;
1560
7a9e9987 1561 WARN_ON(!sblock->pagev[0]->page);
b5d67f64
SB		 1562 if (is_metadata) {
1563 struct btrfs_header *h;
1564
7a9e9987 1565 mapped_buffer = kmap_atomic(sblock->pagev[0]->page);
b5d67f64
SB		 1566 h = (struct btrfs_header *)mapped_buffer;
1567
3cae210f 1568 if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h) ||
17a9be2f 1569 !scrub_check_fsid(h->fsid, sblock->pagev[0]) ||
b5d67f64 1570 memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
442a4f63 1571 BTRFS_UUID_SIZE)) {
b5d67f64 1572 sblock->header_error = 1;
3cae210f 1573 } else if (generation != btrfs_stack_header_generation(h)) {
442a4f63
SB		 1574 sblock->header_error = 1;
1575 sblock->generation_error = 1;
1576 }
b5d67f64
SB		 1577 csum = h->csum;
1578 } else {
1579 if (!have_csum)
1580 return;
a2de733c 1581
7a9e9987 1582 mapped_buffer = kmap_atomic(sblock->pagev[0]->page);
b5d67f64 1583 }
a2de733c 1584
b5d67f64
SB		 1585 for (page_num = 0;;) {
1586 if (page_num == 0 && is_metadata)
b0496686 1587 crc = btrfs_csum_data(
b5d67f64
SB		 1588 ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE,
1589 crc, PAGE_SIZE - BTRFS_CSUM_SIZE);
1590 else
b0496686 1591 crc = btrfs_csum_data(mapped_buffer, crc, PAGE_SIZE);
b5d67f64 1592
9613bebb 1593 kunmap_atomic(mapped_buffer);
b5d67f64
SB		 1594 page_num++;
1595 if (page_num >= sblock->page_count)
1596 break;
7a9e9987 1597 WARN_ON(!sblock->pagev[page_num]->page);
b5d67f64 1598
7a9e9987 1599 mapped_buffer = kmap_atomic(sblock->pagev[page_num]->page);
b5d67f64
SB		 1600 }
1601
1602 btrfs_csum_final(crc, calculated_csum);
1603 if (memcmp(calculated_csum, csum, csum_size))
1604 sblock->checksum_error = 1;
a2de733c
AJ		 1605}
1606
b5d67f64 1607static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
114ab50d 1608 struct scrub_block *sblock_good)
b5d67f64
SB		 1609{
1610 int page_num;
1611 int ret = 0;
96e36920 1612
b5d67f64
SB		 1613 for (page_num = 0; page_num < sblock_bad->page_count; page_num++) {
1614 int ret_sub;
96e36920 1615
b5d67f64
SB		 1616 ret_sub = scrub_repair_page_from_good_copy(sblock_bad,
1617 sblock_good,
114ab50d 1618 page_num, 1);
b5d67f64
SB		 1619 if (ret_sub)
1620 ret = ret_sub;
a2de733c 1621 }
b5d67f64
SB		 1622
1623 return ret;
1624}
1625
1626static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
1627 struct scrub_block *sblock_good,
1628 int page_num, int force_write)
1629{
7a9e9987
SB		 1630 struct scrub_page *page_bad = sblock_bad->pagev[page_num];
1631 struct scrub_page *page_good = sblock_good->pagev[page_num];
b5d67f64 1632
7a9e9987
SB		 1633 BUG_ON(page_bad->page == NULL);
1634 BUG_ON(page_good->page == NULL);
b5d67f64
SB		 1635 if (force_write || sblock_bad->header_error ||
1636 sblock_bad->checksum_error || page_bad->io_error) {
1637 struct bio *bio;
1638 int ret;
b5d67f64 1639
ff023aac 1640 if (!page_bad->dev->bdev) {
efe120a0
FH		 1641 printk_ratelimited(KERN_WARNING "BTRFS: "
1642 "scrub_repair_page_from_good_copy(bdev == NULL) "
1643 "is unexpected!\n");
ff023aac
SB		 1644 return -EIO;
1645 }
1646
9be3395b 1647 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
e627ee7b
TI		 1648 if (!bio)
1649 return -EIO;
442a4f63 1650 bio->bi_bdev = page_bad->dev->bdev;
4f024f37 1651 bio->bi_iter.bi_sector = page_bad->physical >> 9;
b5d67f64
SB		 1652
1653 ret = bio_add_page(bio, page_good->page, PAGE_SIZE, 0);
1654 if (PAGE_SIZE != ret) {
1655 bio_put(bio);
1656 return -EIO;
13db62b7 1657 }
b5d67f64 1658
33879d45 1659 if (btrfsic_submit_bio_wait(WRITE, bio)) {
442a4f63
SB		 1660 btrfs_dev_stat_inc_and_print(page_bad->dev,
1661 BTRFS_DEV_STAT_WRITE_ERRS);
ff023aac
SB		 1662 btrfs_dev_replace_stats_inc(
1663 &sblock_bad->sctx->dev_root->fs_info->
1664 dev_replace.num_write_errors);
442a4f63
SB		 1665 bio_put(bio);
1666 return -EIO;
1667 }
b5d67f64 1668 bio_put(bio);
a2de733c
AJ		 1669 }
1670
b5d67f64
SB		 1671 return 0;
1672}
1673
ff023aac
SB		 1674static void scrub_write_block_to_dev_replace(struct scrub_block *sblock)
1675{
1676 int page_num;
1677
5a6ac9ea
MX		 1678 /*
1679 * This block is used for the check of the parity on the source device,
1680 * so the data needn't be written into the destination device.
1681 */
1682 if (sblock->sparity)
1683 return;
1684
ff023aac
SB		 1685 for (page_num = 0; page_num < sblock->page_count; page_num++) {
1686 int ret;
1687
1688 ret = scrub_write_page_to_dev_replace(sblock, page_num);
1689 if (ret)
1690 btrfs_dev_replace_stats_inc(
1691 &sblock->sctx->dev_root->fs_info->dev_replace.
1692 num_write_errors);
1693 }
1694}
1695
1696static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
1697 int page_num)
1698{
1699 struct scrub_page *spage = sblock->pagev[page_num];
1700
1701 BUG_ON(spage->page == NULL);
1702 if (spage->io_error) {
1703 void *mapped_buffer = kmap_atomic(spage->page);
1704
1705 memset(mapped_buffer, 0, PAGE_CACHE_SIZE);
1706 flush_dcache_page(spage->page);
1707 kunmap_atomic(mapped_buffer);
1708 }
1709 return scrub_add_page_to_wr_bio(sblock->sctx, spage);
1710}
1711
1712static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
1713 struct scrub_page *spage)
1714{
1715 struct scrub_wr_ctx *wr_ctx = &sctx->wr_ctx;
1716 struct scrub_bio *sbio;
1717 int ret;
1718
1719 mutex_lock(&wr_ctx->wr_lock);
1720again:
1721 if (!wr_ctx->wr_curr_bio) {
1722 wr_ctx->wr_curr_bio = kzalloc(sizeof(*wr_ctx->wr_curr_bio),
1723 GFP_NOFS);
1724 if (!wr_ctx->wr_curr_bio) {
1725 mutex_unlock(&wr_ctx->wr_lock);
1726 return -ENOMEM;
1727 }
1728 wr_ctx->wr_curr_bio->sctx = sctx;
1729 wr_ctx->wr_curr_bio->page_count = 0;
1730 }
1731 sbio = wr_ctx->wr_curr_bio;
1732 if (sbio->page_count == 0) {
1733 struct bio *bio;
1734
1735 sbio->physical = spage->physical_for_dev_replace;
1736 sbio->logical = spage->logical;
1737 sbio->dev = wr_ctx->tgtdev;
1738 bio = sbio->bio;
1739 if (!bio) {
9be3395b 1740 bio = btrfs_io_bio_alloc(GFP_NOFS, wr_ctx->pages_per_wr_bio);
ff023aac
SB		 1741 if (!bio) {
1742 mutex_unlock(&wr_ctx->wr_lock);
1743 return -ENOMEM;
1744 }
1745 sbio->bio = bio;
1746 }
1747
1748 bio->bi_private = sbio;
1749 bio->bi_end_io = scrub_wr_bio_end_io;
1750 bio->bi_bdev = sbio->dev->bdev;
4f024f37 1751 bio->bi_iter.bi_sector = sbio->physical >> 9;
ff023aac
SB		 1752 sbio->err = 0;
1753 } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
1754 spage->physical_for_dev_replace ||
1755 sbio->logical + sbio->page_count * PAGE_SIZE !=
1756 spage->logical) {
1757 scrub_wr_submit(sctx);
1758 goto again;
1759 }
1760
1761 ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
1762 if (ret != PAGE_SIZE) {
1763 if (sbio->page_count < 1) {
1764 bio_put(sbio->bio);
1765 sbio->bio = NULL;
1766 mutex_unlock(&wr_ctx->wr_lock);
1767 return -EIO;
1768 }
1769 scrub_wr_submit(sctx);
1770 goto again;
1771 }
1772
1773 sbio->pagev[sbio->page_count] = spage;
1774 scrub_page_get(spage);
1775 sbio->page_count++;
1776 if (sbio->page_count == wr_ctx->pages_per_wr_bio)
1777 scrub_wr_submit(sctx);
1778 mutex_unlock(&wr_ctx->wr_lock);
1779
1780 return 0;
1781}
1782
1783static void scrub_wr_submit(struct scrub_ctx *sctx)
1784{
1785 struct scrub_wr_ctx *wr_ctx = &sctx->wr_ctx;
1786 struct scrub_bio *sbio;
1787
1788 if (!wr_ctx->wr_curr_bio)
1789 return;
1790
1791 sbio = wr_ctx->wr_curr_bio;
1792 wr_ctx->wr_curr_bio = NULL;
1793 WARN_ON(!sbio->bio->bi_bdev);
1794 scrub_pending_bio_inc(sctx);
1795 /* process all writes in a single worker thread. Then the block layer
1796 * orders the requests before sending them to the driver which
1797 * doubled the write performance on spinning disks when measured
1798 * with Linux 3.5 */
1799 btrfsic_submit_bio(WRITE, sbio->bio);
1800}
1801
1802static void scrub_wr_bio_end_io(struct bio *bio, int err)
1803{
1804 struct scrub_bio *sbio = bio->bi_private;
1805 struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;
1806
1807 sbio->err = err;
1808 sbio->bio = bio;
1809
9e0af237
LB		 1810 btrfs_init_work(&sbio->work, btrfs_scrubwrc_helper,
1811 scrub_wr_bio_end_io_worker, NULL, NULL);
0339ef2f 1812 btrfs_queue_work(fs_info->scrub_wr_completion_workers, &sbio->work);
ff023aac
SB		 1813}
1814
1815static void scrub_wr_bio_end_io_worker(struct btrfs_work *work)
1816{
1817 struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
1818 struct scrub_ctx *sctx = sbio->sctx;
1819 int i;
1820
1821 WARN_ON(sbio->page_count > SCRUB_PAGES_PER_WR_BIO);
1822 if (sbio->err) {
1823 struct btrfs_dev_replace *dev_replace =
1824 &sbio->sctx->dev_root->fs_info->dev_replace;
1825
1826 for (i = 0; i < sbio->page_count; i++) {
1827 struct scrub_page *spage = sbio->pagev[i];
1828
1829 spage->io_error = 1;
1830 btrfs_dev_replace_stats_inc(&dev_replace->
1831 num_write_errors);
1832 }
1833 }
1834
1835 for (i = 0; i < sbio->page_count; i++)
1836 scrub_page_put(sbio->pagev[i]);
1837
1838 bio_put(sbio->bio);
1839 kfree(sbio);
1840 scrub_pending_bio_dec(sctx);
1841}
1842
1843static int scrub_checksum(struct scrub_block *sblock)
b5d67f64
SB		 1844{
1845 u64 flags;
1846 int ret;
1847
7a9e9987
SB		 1848 WARN_ON(sblock->page_count < 1);
1849 flags = sblock->pagev[0]->flags;
b5d67f64
SB		 1850 ret = 0;
1851 if (flags & BTRFS_EXTENT_FLAG_DATA)
1852 ret = scrub_checksum_data(sblock);
1853 else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)
1854 ret = scrub_checksum_tree_block(sblock);
1855 else if (flags & BTRFS_EXTENT_FLAG_SUPER)
1856 (void)scrub_checksum_super(sblock);
1857 else
1858 WARN_ON(1);
1859 if (ret)
1860 scrub_handle_errored_block(sblock);
ff023aac
SB		 1861
1862 return ret;
a2de733c
AJ		 1863}
1864
b5d67f64 1865static int scrub_checksum_data(struct scrub_block *sblock)
a2de733c 1866{
d9d181c1 1867 struct scrub_ctx *sctx = sblock->sctx;
a2de733c 1868 u8 csum[BTRFS_CSUM_SIZE];
b5d67f64
SB		 1869 u8 *on_disk_csum;
1870 struct page *page;
1871 void *buffer;
a2de733c
AJ		 1872 u32 crc = ~(u32)0;
1873 int fail = 0;
b5d67f64
SB		 1874 u64 len;
1875 int index;
a2de733c 1876
b5d67f64 1877 BUG_ON(sblock->page_count < 1);
7a9e9987 1878 if (!sblock->pagev[0]->have_csum)
a2de733c
AJ		 1879 return 0;
1880
7a9e9987
SB		 1881 on_disk_csum = sblock->pagev[0]->csum;
1882 page = sblock->pagev[0]->page;
9613bebb 1883 buffer = kmap_atomic(page);
b5d67f64 1884
d9d181c1 1885 len = sctx->sectorsize;
b5d67f64
SB		 1886 index = 0;
1887 for (;;) {
1888 u64 l = min_t(u64, len, PAGE_SIZE);
1889
b0496686 1890 crc = btrfs_csum_data(buffer, crc, l);
9613bebb 1891 kunmap_atomic(buffer);
b5d67f64
SB		 1892 len -= l;
1893 if (len == 0)
1894 break;
1895 index++;
1896 BUG_ON(index >= sblock->page_count);
7a9e9987
SB		 1897 BUG_ON(!sblock->pagev[index]->page);
1898 page = sblock->pagev[index]->page;
9613bebb 1899 buffer = kmap_atomic(page);
b5d67f64
SB		 1900 }
1901
a2de733c 1902 btrfs_csum_final(crc, csum);
d9d181c1 1903 if (memcmp(csum, on_disk_csum, sctx->csum_size))
a2de733c
AJ		 1904 fail = 1;
1905
a2de733c
AJ		 1906 return fail;
1907}
1908
b5d67f64 1909static int scrub_checksum_tree_block(struct scrub_block *sblock)
a2de733c 1910{
d9d181c1 1911 struct scrub_ctx *sctx = sblock->sctx;
a2de733c 1912 struct btrfs_header *h;
a36cf8b8 1913 struct btrfs_root *root = sctx->dev_root;
a2de733c 1914 struct btrfs_fs_info *fs_info = root->fs_info;
b5d67f64
SB		 1915 u8 calculated_csum[BTRFS_CSUM_SIZE];
1916 u8 on_disk_csum[BTRFS_CSUM_SIZE];
1917 struct page *page;
1918 void *mapped_buffer;
1919 u64 mapped_size;
1920 void *p;
a2de733c
AJ		 1921 u32 crc = ~(u32)0;
1922 int fail = 0;
1923 int crc_fail = 0;
b5d67f64
SB		 1924 u64 len;
1925 int index;
1926
1927 BUG_ON(sblock->page_count < 1);
7a9e9987 1928 page = sblock->pagev[0]->page;
9613bebb 1929 mapped_buffer = kmap_atomic(page);
b5d67f64 1930 h = (struct btrfs_header *)mapped_buffer;
d9d181c1 1931 memcpy(on_disk_csum, h->csum, sctx->csum_size);
a2de733c
AJ		 1932
1933 /*
1934 * we don't use the getter functions here, as we
1935 * a) don't have an extent buffer and
1936 * b) the page is already kmapped
1937 */
a2de733c 1938
3cae210f 1939 if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h))
a2de733c
AJ		 1940 ++fail;
1941
3cae210f 1942 if (sblock->pagev[0]->generation != btrfs_stack_header_generation(h))
a2de733c
AJ		 1943 ++fail;
1944
17a9be2f 1945 if (!scrub_check_fsid(h->fsid, sblock->pagev[0]))
a2de733c
AJ		 1946 ++fail;
1947
1948 if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
1949 BTRFS_UUID_SIZE))
1950 ++fail;
1951
d9d181c1 1952 len = sctx->nodesize - BTRFS_CSUM_SIZE;
b5d67f64
SB		 1953 mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
1954 p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
1955 index = 0;
1956 for (;;) {
1957 u64 l = min_t(u64, len, mapped_size);
1958
b0496686 1959 crc = btrfs_csum_data(p, crc, l);
9613bebb 1960 kunmap_atomic(mapped_buffer);
b5d67f64
SB		 1961 len -= l;
1962 if (len == 0)
1963 break;
1964 index++;
1965 BUG_ON(index >= sblock->page_count);
7a9e9987
SB		 1966 BUG_ON(!sblock->pagev[index]->page);
1967 page = sblock->pagev[index]->page;
9613bebb 1968 mapped_buffer = kmap_atomic(page);
b5d67f64
SB		 1969 mapped_size = PAGE_SIZE;
1970 p = mapped_buffer;
1971 }
1972
1973 btrfs_csum_final(crc, calculated_csum);
d9d181c1 1974 if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
a2de733c
AJ		 1975 ++crc_fail;
1976
a2de733c
AJ		 1977 return fail || crc_fail;
1978}
1979
b5d67f64 1980static int scrub_checksum_super(struct scrub_block *sblock)
a2de733c
AJ		 1981{
1982 struct btrfs_super_block *s;
d9d181c1 1983 struct scrub_ctx *sctx = sblock->sctx;
b5d67f64
SB		 1984 u8 calculated_csum[BTRFS_CSUM_SIZE];
1985 u8 on_disk_csum[BTRFS_CSUM_SIZE];
1986 struct page *page;
1987 void *mapped_buffer;
1988 u64 mapped_size;
1989 void *p;
a2de733c 1990 u32 crc = ~(u32)0;
442a4f63
SB		 1991 int fail_gen = 0;
1992 int fail_cor = 0;
b5d67f64
SB		 1993 u64 len;
1994 int index;
a2de733c 1995
b5d67f64 1996 BUG_ON(sblock->page_count < 1);
7a9e9987 1997 page = sblock->pagev[0]->page;
9613bebb 1998 mapped_buffer = kmap_atomic(page);
b5d67f64 1999 s = (struct btrfs_super_block *)mapped_buffer;
d9d181c1 2000 memcpy(on_disk_csum, s->csum, sctx->csum_size);
a2de733c 2001
3cae210f 2002 if (sblock->pagev[0]->logical != btrfs_super_bytenr(s))
442a4f63 2003 ++fail_cor;
a2de733c 2004
3cae210f 2005 if (sblock->pagev[0]->generation != btrfs_super_generation(s))
442a4f63 2006 ++fail_gen;
a2de733c 2007
17a9be2f 2008 if (!scrub_check_fsid(s->fsid, sblock->pagev[0]))
442a4f63 2009 ++fail_cor;
a2de733c 2010
b5d67f64
SB		 2011 len = BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE;
2012 mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
2013 p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
2014 index = 0;
2015 for (;;) {
2016 u64 l = min_t(u64, len, mapped_size);
2017
b0496686 2018 crc = btrfs_csum_data(p, crc, l);
9613bebb 2019 kunmap_atomic(mapped_buffer);
b5d67f64
SB		 2020 len -= l;
2021 if (len == 0)
2022 break;
2023 index++;
2024 BUG_ON(index >= sblock->page_count);
7a9e9987
SB		 2025 BUG_ON(!sblock->pagev[index]->page);
2026 page = sblock->pagev[index]->page;
9613bebb 2027 mapped_buffer = kmap_atomic(page);
b5d67f64
SB		 2028 mapped_size = PAGE_SIZE;
2029 p = mapped_buffer;
2030 }
2031
2032 btrfs_csum_final(crc, calculated_csum);
d9d181c1 2033 if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
442a4f63 2034 ++fail_cor;
a2de733c 2035
442a4f63 2036 if (fail_cor + fail_gen) {
a2de733c
AJ		 2037 /*
2038 * if we find an error in a super block, we just report it.
2039 * They will get written with the next transaction commit
2040 * anyway
2041 */
d9d181c1
SB		 2042 spin_lock(&sctx->stat_lock);
2043 ++sctx->stat.super_errors;
2044 spin_unlock(&sctx->stat_lock);
442a4f63 2045 if (fail_cor)
7a9e9987 2046 btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
442a4f63
SB		 2047 BTRFS_DEV_STAT_CORRUPTION_ERRS);
2048 else
7a9e9987 2049 btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
442a4f63 2050 BTRFS_DEV_STAT_GENERATION_ERRS);
a2de733c
AJ		 2051 }
2052
442a4f63 2053 return fail_cor + fail_gen;
a2de733c
AJ		 2054}
2055
b5d67f64
SB		 2056static void scrub_block_get(struct scrub_block *sblock)
2057{
57019345 2058 atomic_inc(&sblock->refs);
b5d67f64
SB		 2059}
2060
2061static void scrub_block_put(struct scrub_block *sblock)
2062{
57019345 2063 if (atomic_dec_and_test(&sblock->refs)) {
b5d67f64
SB		 2064 int i;
2065
5a6ac9ea
MX		 2066 if (sblock->sparity)
2067 scrub_parity_put(sblock->sparity);
2068
b5d67f64 2069 for (i = 0; i < sblock->page_count; i++)
7a9e9987 2070 scrub_page_put(sblock->pagev[i]);
b5d67f64
SB		 2071 kfree(sblock);
2072 }
2073}
2074
7a9e9987
SB		 2075static void scrub_page_get(struct scrub_page *spage)
2076{
57019345 2077 atomic_inc(&spage->refs);
7a9e9987
SB		 2078}
2079
2080static void scrub_page_put(struct scrub_page *spage)
2081{
57019345 2082 if (atomic_dec_and_test(&spage->refs)) {
7a9e9987
SB		 2083 if (spage->page)
2084 __free_page(spage->page);
2085 kfree(spage);
2086 }
2087}
2088
d9d181c1 2089static void scrub_submit(struct scrub_ctx *sctx)
a2de733c
AJ		 2090{
2091 struct scrub_bio *sbio;
2092
d9d181c1 2093 if (sctx->curr == -1)
1623edeb 2094 return;
a2de733c 2095
d9d181c1
SB		 2096 sbio = sctx->bios[sctx->curr];
2097 sctx->curr = -1;
b6bfebc1 2098 scrub_pending_bio_inc(sctx);
a2de733c 2099
ff023aac
SB		 2100 if (!sbio->bio->bi_bdev) {
2101 /*
2102 * this case should not happen. If btrfs_map_block() is
2103 * wrong, it could happen for dev-replace operations on
2104 * missing devices when no mirrors are available, but in
2105 * this case it should already fail the mount.
2106 * This case is handled correctly (but _very_ slowly).
2107 */
2108 printk_ratelimited(KERN_WARNING
efe120a0 2109 "BTRFS: scrub_submit(bio bdev == NULL) is unexpected!\n");
ff023aac
SB		 2110 bio_endio(sbio->bio, -EIO);
2111 } else {
2112 btrfsic_submit_bio(READ, sbio->bio);
2113 }
a2de733c
AJ		 2114}
2115
ff023aac
SB		 2116static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
2117 struct scrub_page *spage)
a2de733c 2118{
b5d67f64 2119 struct scrub_block *sblock = spage->sblock;
a2de733c 2120 struct scrub_bio *sbio;
69f4cb52 2121 int ret;
a2de733c
AJ		 2122
2123again:
2124 /*
2125 * grab a fresh bio or wait for one to become available
2126 */
d9d181c1
SB		 2127 while (sctx->curr == -1) {
2128 spin_lock(&sctx->list_lock);
2129 sctx->curr = sctx->first_free;
2130 if (sctx->curr != -1) {
2131 sctx->first_free = sctx->bios[sctx->curr]->next_free;
2132 sctx->bios[sctx->curr]->next_free = -1;
2133 sctx->bios[sctx->curr]->page_count = 0;
2134 spin_unlock(&sctx->list_lock);
a2de733c 2135 } else {
d9d181c1
SB		 2136 spin_unlock(&sctx->list_lock);
2137 wait_event(sctx->list_wait, sctx->first_free != -1);
a2de733c
AJ		 2138 }
2139 }
d9d181c1 2140 sbio = sctx->bios[sctx->curr];
b5d67f64 2141 if (sbio->page_count == 0) {
69f4cb52
AJ		 2142 struct bio *bio;
2143
b5d67f64
SB		 2144 sbio->physical = spage->physical;
2145 sbio->logical = spage->logical;
a36cf8b8 2146 sbio->dev = spage->dev;
b5d67f64
SB		 2147 bio = sbio->bio;
2148 if (!bio) {
9be3395b 2149 bio = btrfs_io_bio_alloc(GFP_NOFS, sctx->pages_per_rd_bio);
b5d67f64
SB		 2150 if (!bio)
2151 return -ENOMEM;
2152 sbio->bio = bio;
2153 }
69f4cb52
AJ		 2154
2155 bio->bi_private = sbio;
2156 bio->bi_end_io = scrub_bio_end_io;
a36cf8b8 2157 bio->bi_bdev = sbio->dev->bdev;
4f024f37 2158 bio->bi_iter.bi_sector = sbio->physical >> 9;
69f4cb52 2159 sbio->err = 0;
b5d67f64
SB		 2160 } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
2161 spage->physical ||
2162 sbio->logical + sbio->page_count * PAGE_SIZE !=
a36cf8b8
SB		 2163 spage->logical ||
2164 sbio->dev != spage->dev) {
d9d181c1 2165 scrub_submit(sctx);
a2de733c
AJ		 2166 goto again;
2167 }
69f4cb52 2168
b5d67f64
SB		 2169 sbio->pagev[sbio->page_count] = spage;
2170 ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
2171 if (ret != PAGE_SIZE) {
2172 if (sbio->page_count < 1) {
2173 bio_put(sbio->bio);
2174 sbio->bio = NULL;
2175 return -EIO;
2176 }
d9d181c1 2177 scrub_submit(sctx);
69f4cb52
AJ		 2178 goto again;
2179 }
2180
ff023aac 2181 scrub_block_get(sblock); /* one for the page added to the bio */
b5d67f64
SB		 2182 atomic_inc(&sblock->outstanding_pages);
2183 sbio->page_count++;
ff023aac 2184 if (sbio->page_count == sctx->pages_per_rd_bio)
d9d181c1 2185 scrub_submit(sctx);
b5d67f64
SB		 2186
2187 return 0;
2188}
2189
d9d181c1 2190static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
a36cf8b8 2191 u64 physical, struct btrfs_device *dev, u64 flags,
ff023aac
SB		 2192 u64 gen, int mirror_num, u8 *csum, int force,
2193 u64 physical_for_dev_replace)
b5d67f64
SB		 2194{
2195 struct scrub_block *sblock;
2196 int index;
2197
2198 sblock = kzalloc(sizeof(*sblock), GFP_NOFS);
2199 if (!sblock) {
d9d181c1
SB		 2200 spin_lock(&sctx->stat_lock);
2201 sctx->stat.malloc_errors++;
2202 spin_unlock(&sctx->stat_lock);
b5d67f64 2203 return -ENOMEM;
a2de733c 2204 }
b5d67f64 2205
7a9e9987
SB		 2206 /* one ref inside this function, plus one for each page added to
2207 * a bio later on */
57019345 2208 atomic_set(&sblock->refs, 1);
d9d181c1 2209 sblock->sctx = sctx;
b5d67f64
SB		 2210 sblock->no_io_error_seen = 1;
2211
2212 for (index = 0; len > 0; index++) {
7a9e9987 2213 struct scrub_page *spage;
b5d67f64
SB		 2214 u64 l = min_t(u64, len, PAGE_SIZE);
2215
7a9e9987
SB		 2216 spage = kzalloc(sizeof(*spage), GFP_NOFS);
2217 if (!spage) {
2218leave_nomem:
d9d181c1
SB		 2219 spin_lock(&sctx->stat_lock);
2220 sctx->stat.malloc_errors++;
2221 spin_unlock(&sctx->stat_lock);
7a9e9987 2222 scrub_block_put(sblock);
b5d67f64
SB		 2223 return -ENOMEM;
2224 }
7a9e9987
SB		 2225 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
2226 scrub_page_get(spage);
2227 sblock->pagev[index] = spage;
b5d67f64 2228 spage->sblock = sblock;
a36cf8b8 2229 spage->dev = dev;
b5d67f64
SB		 2230 spage->flags = flags;
2231 spage->generation = gen;
2232 spage->logical = logical;
2233 spage->physical = physical;
ff023aac 2234 spage->physical_for_dev_replace = physical_for_dev_replace;
b5d67f64
SB		 2235 spage->mirror_num = mirror_num;
2236 if (csum) {
2237 spage->have_csum = 1;
d9d181c1 2238 memcpy(spage->csum, csum, sctx->csum_size);
b5d67f64
SB		 2239 } else {
2240 spage->have_csum = 0;
2241 }
2242 sblock->page_count++;
7a9e9987
SB		 2243 spage->page = alloc_page(GFP_NOFS);
2244 if (!spage->page)
2245 goto leave_nomem;
b5d67f64
SB		 2246 len -= l;
2247 logical += l;
2248 physical += l;
ff023aac 2249 physical_for_dev_replace += l;
b5d67f64
SB		 2250 }
2251
7a9e9987 2252 WARN_ON(sblock->page_count == 0);
b5d67f64 2253 for (index = 0; index < sblock->page_count; index++) {
7a9e9987 2254 struct scrub_page *spage = sblock->pagev[index];
1bc87793
AJ		 2255 int ret;
2256
ff023aac 2257 ret = scrub_add_page_to_rd_bio(sctx, spage);
b5d67f64
SB		 2258 if (ret) {
2259 scrub_block_put(sblock);
1bc87793 2260 return ret;
b5d67f64 2261 }
1bc87793 2262 }
a2de733c 2263
b5d67f64 2264 if (force)
d9d181c1 2265 scrub_submit(sctx);
a2de733c 2266
b5d67f64
SB		 2267 /* last one frees, either here or in bio completion for last page */
2268 scrub_block_put(sblock);
a2de733c
AJ		 2269 return 0;
2270}
2271
b5d67f64
SB		 2272static void scrub_bio_end_io(struct bio *bio, int err)
2273{
2274 struct scrub_bio *sbio = bio->bi_private;
a36cf8b8 2275 struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;
b5d67f64
SB		 2276
2277 sbio->err = err;
2278 sbio->bio = bio;
2279
0339ef2f 2280 btrfs_queue_work(fs_info->scrub_workers, &sbio->work);
b5d67f64
SB		 2281}
2282
2283static void scrub_bio_end_io_worker(struct btrfs_work *work)
2284{
2285 struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
d9d181c1 2286 struct scrub_ctx *sctx = sbio->sctx;
b5d67f64
SB		 2287 int i;
2288
ff023aac 2289 BUG_ON(sbio->page_count > SCRUB_PAGES_PER_RD_BIO);
b5d67f64
SB		 2290 if (sbio->err) {
2291 for (i = 0; i < sbio->page_count; i++) {
2292 struct scrub_page *spage = sbio->pagev[i];
2293
2294 spage->io_error = 1;
2295 spage->sblock->no_io_error_seen = 0;
2296 }
2297 }
2298
2299 /* now complete the scrub_block items that have all pages completed */
2300 for (i = 0; i < sbio->page_count; i++) {
2301 struct scrub_page *spage = sbio->pagev[i];
2302 struct scrub_block *sblock = spage->sblock;
2303
2304 if (atomic_dec_and_test(&sblock->outstanding_pages))
2305 scrub_block_complete(sblock);
2306 scrub_block_put(sblock);
2307 }
2308
b5d67f64
SB		 2309 bio_put(sbio->bio);
2310 sbio->bio = NULL;
d9d181c1
SB		 2311 spin_lock(&sctx->list_lock);
2312 sbio->next_free = sctx->first_free;
2313 sctx->first_free = sbio->index;
2314 spin_unlock(&sctx->list_lock);
ff023aac
SB		 2315
2316 if (sctx->is_dev_replace &&
2317 atomic_read(&sctx->wr_ctx.flush_all_writes)) {
2318 mutex_lock(&sctx->wr_ctx.wr_lock);
2319 scrub_wr_submit(sctx);
2320 mutex_unlock(&sctx->wr_ctx.wr_lock);
2321 }
2322
b6bfebc1 2323 scrub_pending_bio_dec(sctx);
b5d67f64
SB		 2324}
2325
5a6ac9ea
MX		 2326static inline void __scrub_mark_bitmap(struct scrub_parity *sparity,
2327 unsigned long *bitmap,
2328 u64 start, u64 len)
2329{
9d644a62 2330 u32 offset;
5a6ac9ea
MX		 2331 int nsectors;
2332 int sectorsize = sparity->sctx->dev_root->sectorsize;
2333
2334 if (len >= sparity->stripe_len) {
2335 bitmap_set(bitmap, 0, sparity->nsectors);
2336 return;
2337 }
2338
2339 start -= sparity->logic_start;
47c5713f 2340 start = div_u64_rem(start, sparity->stripe_len, &offset);
5a6ac9ea
MX		 2341 offset /= sectorsize;
2342 nsectors = (int)len / sectorsize;
2343
2344 if (offset + nsectors <= sparity->nsectors) {
2345 bitmap_set(bitmap, offset, nsectors);
2346 return;
2347 }
2348
2349 bitmap_set(bitmap, offset, sparity->nsectors - offset);
2350 bitmap_set(bitmap, 0, nsectors - (sparity->nsectors - offset));
2351}
2352
2353static inline void scrub_parity_mark_sectors_error(struct scrub_parity *sparity,
2354 u64 start, u64 len)
2355{
2356 __scrub_mark_bitmap(sparity, sparity->ebitmap, start, len);
2357}
2358
2359static inline void scrub_parity_mark_sectors_data(struct scrub_parity *sparity,
2360 u64 start, u64 len)
2361{
2362 __scrub_mark_bitmap(sparity, sparity->dbitmap, start, len);
2363}
2364
b5d67f64
SB		 2365static void scrub_block_complete(struct scrub_block *sblock)
2366{
5a6ac9ea
MX		 2367 int corrupted = 0;
2368
ff023aac 2369 if (!sblock->no_io_error_seen) {
5a6ac9ea 2370 corrupted = 1;
b5d67f64 2371 scrub_handle_errored_block(sblock);
ff023aac
SB		 2372 } else {
2373 /*
2374 * if has checksum error, write via repair mechanism in
2375 * dev replace case, otherwise write here in dev replace
2376 * case.
2377 */
5a6ac9ea
MX		 2378 corrupted = scrub_checksum(sblock);
2379 if (!corrupted && sblock->sctx->is_dev_replace)
ff023aac
SB		 2380 scrub_write_block_to_dev_replace(sblock);
2381 }
5a6ac9ea
MX		 2382
2383 if (sblock->sparity && corrupted && !sblock->data_corrected) {
2384 u64 start = sblock->pagev[0]->logical;
2385 u64 end = sblock->pagev[sblock->page_count - 1]->logical +
2386 PAGE_SIZE;
2387
2388 scrub_parity_mark_sectors_error(sblock->sparity,
2389 start, end - start);
2390 }
b5d67f64
SB		 2391}
2392
d9d181c1 2393static int scrub_find_csum(struct scrub_ctx *sctx, u64 logical, u64 len,
a2de733c
AJ		 2394 u8 *csum)
2395{
2396 struct btrfs_ordered_sum *sum = NULL;
f51a4a18 2397 unsigned long index;
a2de733c 2398 unsigned long num_sectors;
a2de733c 2399
d9d181c1
SB		 2400 while (!list_empty(&sctx->csum_list)) {
2401 sum = list_first_entry(&sctx->csum_list,
a2de733c
AJ		 2402 struct btrfs_ordered_sum, list);
2403 if (sum->bytenr > logical)
2404 return 0;
2405 if (sum->bytenr + sum->len > logical)
2406 break;
2407
d9d181c1 2408 ++sctx->stat.csum_discards;
a2de733c
AJ		 2409 list_del(&sum->list);
2410 kfree(sum);
2411 sum = NULL;
2412 }
2413 if (!sum)
2414 return 0;
2415
f51a4a18 2416 index = ((u32)(logical - sum->bytenr)) / sctx->sectorsize;
d9d181c1 2417 num_sectors = sum->len / sctx->sectorsize;
f51a4a18
MX		 2418 memcpy(csum, sum->sums + index, sctx->csum_size);
2419 if (index == num_sectors - 1) {
a2de733c
AJ		 2420 list_del(&sum->list);
2421 kfree(sum);
2422 }
f51a4a18 2423 return 1;
a2de733c
AJ		 2424}
2425
2426/* scrub extent tries to collect up to 64 kB for each bio */
d9d181c1 2427static int scrub_extent(struct scrub_ctx *sctx, u64 logical, u64 len,
a36cf8b8 2428 u64 physical, struct btrfs_device *dev, u64 flags,
ff023aac 2429 u64 gen, int mirror_num, u64 physical_for_dev_replace)
a2de733c
AJ		 2430{
2431 int ret;
2432 u8 csum[BTRFS_CSUM_SIZE];
b5d67f64
SB		 2433 u32 blocksize;
2434
2435 if (flags & BTRFS_EXTENT_FLAG_DATA) {
d9d181c1
SB		 2436 blocksize = sctx->sectorsize;
2437 spin_lock(&sctx->stat_lock);
2438 sctx->stat.data_extents_scrubbed++;
2439 sctx->stat.data_bytes_scrubbed += len;
2440 spin_unlock(&sctx->stat_lock);
b5d67f64 2441 } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
d9d181c1
SB		 2442 blocksize = sctx->nodesize;
2443 spin_lock(&sctx->stat_lock);
2444 sctx->stat.tree_extents_scrubbed++;
2445 sctx->stat.tree_bytes_scrubbed += len;
2446 spin_unlock(&sctx->stat_lock);
b5d67f64 2447 } else {
d9d181c1 2448 blocksize = sctx->sectorsize;
ff023aac 2449 WARN_ON(1);
b5d67f64 2450 }
a2de733c
AJ		 2451
2452 while (len) {
b5d67f64 2453 u64 l = min_t(u64, len, blocksize);
a2de733c
AJ		 2454 int have_csum = 0;
2455
2456 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2457 /* push csums to sbio */
d9d181c1 2458 have_csum = scrub_find_csum(sctx, logical, l, csum);
a2de733c 2459 if (have_csum == 0)
d9d181c1 2460 ++sctx->stat.no_csum;
ff023aac
SB		 2461 if (sctx->is_dev_replace && !have_csum) {
2462 ret = copy_nocow_pages(sctx, logical, l,
2463 mirror_num,
2464 physical_for_dev_replace);
2465 goto behind_scrub_pages;
2466 }
a2de733c 2467 }
a36cf8b8 2468 ret = scrub_pages(sctx, logical, l, physical, dev, flags, gen,
ff023aac
SB		 2469 mirror_num, have_csum ? csum : NULL, 0,
2470 physical_for_dev_replace);
2471behind_scrub_pages:
a2de733c
AJ		 2472 if (ret)
2473 return ret;
2474 len -= l;
2475 logical += l;
2476 physical += l;
ff023aac 2477 physical_for_dev_replace += l;
a2de733c
AJ		 2478 }
2479 return 0;
2480}
2481
5a6ac9ea
MX		 2482static int scrub_pages_for_parity(struct scrub_parity *sparity,
2483 u64 logical, u64 len,
2484 u64 physical, struct btrfs_device *dev,
2485 u64 flags, u64 gen, int mirror_num, u8 *csum)
2486{
2487 struct scrub_ctx *sctx = sparity->sctx;
2488 struct scrub_block *sblock;
2489 int index;
2490
2491 sblock = kzalloc(sizeof(*sblock), GFP_NOFS);
2492 if (!sblock) {
2493 spin_lock(&sctx->stat_lock);
2494 sctx->stat.malloc_errors++;
2495 spin_unlock(&sctx->stat_lock);
2496 return -ENOMEM;
2497 }
2498
2499 /* one ref inside this function, plus one for each page added to
2500 * a bio later on */
57019345 2501 atomic_set(&sblock->refs, 1);
5a6ac9ea
MX		 2502 sblock->sctx = sctx;
2503 sblock->no_io_error_seen = 1;
2504 sblock->sparity = sparity;
2505 scrub_parity_get(sparity);
2506
2507 for (index = 0; len > 0; index++) {
2508 struct scrub_page *spage;
2509 u64 l = min_t(u64, len, PAGE_SIZE);
2510
2511 spage = kzalloc(sizeof(*spage), GFP_NOFS);
2512 if (!spage) {
2513leave_nomem:
2514 spin_lock(&sctx->stat_lock);
2515 sctx->stat.malloc_errors++;
2516 spin_unlock(&sctx->stat_lock);
2517 scrub_block_put(sblock);
2518 return -ENOMEM;
2519 }
2520 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
2521 /* For scrub block */
2522 scrub_page_get(spage);
2523 sblock->pagev[index] = spage;
2524 /* For scrub parity */
2525 scrub_page_get(spage);
2526 list_add_tail(&spage->list, &sparity->spages);
2527 spage->sblock = sblock;
2528 spage->dev = dev;
2529 spage->flags = flags;
2530 spage->generation = gen;
2531 spage->logical = logical;
2532 spage->physical = physical;
2533 spage->mirror_num = mirror_num;
2534 if (csum) {
2535 spage->have_csum = 1;
2536 memcpy(spage->csum, csum, sctx->csum_size);
2537 } else {
2538 spage->have_csum = 0;
2539 }
2540 sblock->page_count++;
2541 spage->page = alloc_page(GFP_NOFS);
2542 if (!spage->page)
2543 goto leave_nomem;
2544 len -= l;
2545 logical += l;
2546 physical += l;
2547 }
2548
2549 WARN_ON(sblock->page_count == 0);
2550 for (index = 0; index < sblock->page_count; index++) {
2551 struct scrub_page *spage = sblock->pagev[index];
2552 int ret;
2553
2554 ret = scrub_add_page_to_rd_bio(sctx, spage);
2555 if (ret) {
2556 scrub_block_put(sblock);
2557 return ret;
2558 }
2559 }
2560
2561 /* last one frees, either here or in bio completion for last page */
2562 scrub_block_put(sblock);
2563 return 0;
2564}
2565
2566static int scrub_extent_for_parity(struct scrub_parity *sparity,
2567 u64 logical, u64 len,
2568 u64 physical, struct btrfs_device *dev,
2569 u64 flags, u64 gen, int mirror_num)
2570{
2571 struct scrub_ctx *sctx = sparity->sctx;
2572 int ret;
2573 u8 csum[BTRFS_CSUM_SIZE];
2574 u32 blocksize;
2575
2576 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2577 blocksize = sctx->sectorsize;
2578 } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
2579 blocksize = sctx->nodesize;
2580 } else {
2581 blocksize = sctx->sectorsize;
2582 WARN_ON(1);
2583 }
2584
2585 while (len) {
2586 u64 l = min_t(u64, len, blocksize);
2587 int have_csum = 0;
2588
2589 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2590 /* push csums to sbio */
2591 have_csum = scrub_find_csum(sctx, logical, l, csum);
2592 if (have_csum == 0)
2593 goto skip;
2594 }
2595 ret = scrub_pages_for_parity(sparity, logical, l, physical, dev,
2596 flags, gen, mirror_num,
2597 have_csum ? csum : NULL);
5a6ac9ea
MX		 2598 if (ret)
2599 return ret;
6b6d24b3 2600skip:
5a6ac9ea
MX		 2601 len -= l;
2602 logical += l;
2603 physical += l;
2604 }
2605 return 0;
2606}
2607
3b080b25
WS		 2608/*
2609 * Given a physical address, this will calculate it's
2610 * logical offset. if this is a parity stripe, it will return
2611 * the most left data stripe's logical offset.
2612 *
2613 * return 0 if it is a data stripe, 1 means parity stripe.
2614 */
2615static int get_raid56_logic_offset(u64 physical, int num,
5a6ac9ea
MX		 2616 struct map_lookup *map, u64 *offset,
2617 u64 *stripe_start)
3b080b25
WS		 2618{
2619 int i;
2620 int j = 0;
2621 u64 stripe_nr;
2622 u64 last_offset;
9d644a62
DS		 2623 u32 stripe_index;
2624 u32 rot;
3b080b25
WS		 2625
2626 last_offset = (physical - map->stripes[num].physical) *
2627 nr_data_stripes(map);
5a6ac9ea
MX		 2628 if (stripe_start)
2629 *stripe_start = last_offset;
2630
3b080b25
WS		 2631 *offset = last_offset;
2632 for (i = 0; i < nr_data_stripes(map); i++) {
2633 *offset = last_offset + i * map->stripe_len;
2634
b8b93add
DS		 2635 stripe_nr = div_u64(*offset, map->stripe_len);
2636 stripe_nr = div_u64(stripe_nr, nr_data_stripes(map));
3b080b25
WS		 2637
2638 /* Work out the disk rotation on this stripe-set */
47c5713f 2639 stripe_nr = div_u64_rem(stripe_nr, map->num_stripes, &rot);
3b080b25
WS		 2640 /* calculate which stripe this data locates */
2641 rot += i;
e4fbaee2 2642 stripe_index = rot % map->num_stripes;
3b080b25
WS		 2643 if (stripe_index == num)
2644 return 0;
2645 if (stripe_index < num)
2646 j++;
2647 }
2648 *offset = last_offset + j * map->stripe_len;
2649 return 1;
2650}
2651
5a6ac9ea
MX		 2652static void scrub_free_parity(struct scrub_parity *sparity)
2653{
2654 struct scrub_ctx *sctx = sparity->sctx;
2655 struct scrub_page *curr, *next;
2656 int nbits;
2657
2658 nbits = bitmap_weight(sparity->ebitmap, sparity->nsectors);
2659 if (nbits) {
2660 spin_lock(&sctx->stat_lock);
2661 sctx->stat.read_errors += nbits;
2662 sctx->stat.uncorrectable_errors += nbits;
2663 spin_unlock(&sctx->stat_lock);
2664 }
2665
2666 list_for_each_entry_safe(curr, next, &sparity->spages, list) {
2667 list_del_init(&curr->list);
2668 scrub_page_put(curr);
2669 }
2670
2671 kfree(sparity);
2672}
2673
20b2e302
ZL		 2674static void scrub_parity_bio_endio_worker(struct btrfs_work *work)
2675{
2676 struct scrub_parity *sparity = container_of(work, struct scrub_parity,
2677 work);
2678 struct scrub_ctx *sctx = sparity->sctx;
2679
2680 scrub_free_parity(sparity);
2681 scrub_pending_bio_dec(sctx);
2682}
2683
5a6ac9ea
MX		 2684static void scrub_parity_bio_endio(struct bio *bio, int error)
2685{
2686 struct scrub_parity *sparity = (struct scrub_parity *)bio->bi_private;
5a6ac9ea
MX		 2687
2688 if (error)
2689 bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
2690 sparity->nsectors);
2691
5a6ac9ea 2692 bio_put(bio);
20b2e302
ZL		 2693
2694 btrfs_init_work(&sparity->work, btrfs_scrubparity_helper,
2695 scrub_parity_bio_endio_worker, NULL, NULL);
2696 btrfs_queue_work(sparity->sctx->dev_root->fs_info->scrub_parity_workers,
2697 &sparity->work);
5a6ac9ea
MX		 2698}
2699
2700static void scrub_parity_check_and_repair(struct scrub_parity *sparity)
2701{
2702 struct scrub_ctx *sctx = sparity->sctx;
2703 struct bio *bio;
2704 struct btrfs_raid_bio *rbio;
2705 struct scrub_page *spage;
2706 struct btrfs_bio *bbio = NULL;
5a6ac9ea
MX		 2707 u64 length;
2708 int ret;
2709
2710 if (!bitmap_andnot(sparity->dbitmap, sparity->dbitmap, sparity->ebitmap,
2711 sparity->nsectors))
2712 goto out;
2713
a0dd59de 2714 length = sparity->logic_end - sparity->logic_start;
76035976 2715 ret = btrfs_map_sblock(sctx->dev_root->fs_info, WRITE,
5a6ac9ea 2716 sparity->logic_start,
8e5cfb55
ZL		 2717 &length, &bbio, 0, 1);
2718 if (ret || !bbio || !bbio->raid_map)
5a6ac9ea
MX		 2719 goto bbio_out;
2720
2721 bio = btrfs_io_bio_alloc(GFP_NOFS, 0);
2722 if (!bio)
2723 goto bbio_out;
2724
2725 bio->bi_iter.bi_sector = sparity->logic_start >> 9;
2726 bio->bi_private = sparity;
2727 bio->bi_end_io = scrub_parity_bio_endio;
2728
2729 rbio = raid56_parity_alloc_scrub_rbio(sctx->dev_root, bio, bbio,
8e5cfb55 2730 length, sparity->scrub_dev,
5a6ac9ea
MX		 2731 sparity->dbitmap,
2732 sparity->nsectors);
2733 if (!rbio)
2734 goto rbio_out;
2735
2736 list_for_each_entry(spage, &sparity->spages, list)
2737 raid56_parity_add_scrub_pages(rbio, spage->page,
2738 spage->logical);
2739
2740 scrub_pending_bio_inc(sctx);
2741 raid56_parity_submit_scrub_rbio(rbio);
2742 return;
2743
2744rbio_out:
2745 bio_put(bio);
2746bbio_out:
6e9606d2 2747 btrfs_put_bbio(bbio);
5a6ac9ea
MX		 2748 bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
2749 sparity->nsectors);
2750 spin_lock(&sctx->stat_lock);
2751 sctx->stat.malloc_errors++;
2752 spin_unlock(&sctx->stat_lock);
2753out:
2754 scrub_free_parity(sparity);
2755}
2756
2757static inline int scrub_calc_parity_bitmap_len(int nsectors)
2758{
2759 return DIV_ROUND_UP(nsectors, BITS_PER_LONG) * (BITS_PER_LONG / 8);
2760}
2761
2762static void scrub_parity_get(struct scrub_parity *sparity)
2763{
57019345 2764 atomic_inc(&sparity->refs);
5a6ac9ea
MX		 2765}
2766
2767static void scrub_parity_put(struct scrub_parity *sparity)
2768{
57019345 2769 if (!atomic_dec_and_test(&sparity->refs))
5a6ac9ea
MX		 2770 return;
2771
2772 scrub_parity_check_and_repair(sparity);
2773}
2774
2775static noinline_for_stack int scrub_raid56_parity(struct scrub_ctx *sctx,
2776 struct map_lookup *map,
2777 struct btrfs_device *sdev,
2778 struct btrfs_path *path,
2779 u64 logic_start,
2780 u64 logic_end)
2781{
2782 struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
2783 struct btrfs_root *root = fs_info->extent_root;
2784 struct btrfs_root *csum_root = fs_info->csum_root;
2785 struct btrfs_extent_item *extent;
2786 u64 flags;
2787 int ret;
2788 int slot;
2789 struct extent_buffer *l;
2790 struct btrfs_key key;
2791 u64 generation;
2792 u64 extent_logical;
2793 u64 extent_physical;
2794 u64 extent_len;
2795 struct btrfs_device *extent_dev;
2796 struct scrub_parity *sparity;
2797 int nsectors;
2798 int bitmap_len;
2799 int extent_mirror_num;
2800 int stop_loop = 0;
2801
2802 nsectors = map->stripe_len / root->sectorsize;
2803 bitmap_len = scrub_calc_parity_bitmap_len(nsectors);
2804 sparity = kzalloc(sizeof(struct scrub_parity) + 2 * bitmap_len,
2805 GFP_NOFS);
2806 if (!sparity) {
2807 spin_lock(&sctx->stat_lock);
2808 sctx->stat.malloc_errors++;
2809 spin_unlock(&sctx->stat_lock);
2810 return -ENOMEM;
2811 }
2812
2813 sparity->stripe_len = map->stripe_len;
2814 sparity->nsectors = nsectors;
2815 sparity->sctx = sctx;
2816 sparity->scrub_dev = sdev;
2817 sparity->logic_start = logic_start;
2818 sparity->logic_end = logic_end;
57019345 2819 atomic_set(&sparity->refs, 1);
5a6ac9ea
MX		 2820 INIT_LIST_HEAD(&sparity->spages);
2821 sparity->dbitmap = sparity->bitmap;
2822 sparity->ebitmap = (void *)sparity->bitmap + bitmap_len;
2823
2824 ret = 0;
2825 while (logic_start < logic_end) {
2826 if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
2827 key.type = BTRFS_METADATA_ITEM_KEY;
2828 else
2829 key.type = BTRFS_EXTENT_ITEM_KEY;
2830 key.objectid = logic_start;
2831 key.offset = (u64)-1;
2832
2833 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2834 if (ret < 0)
2835 goto out;
2836
2837 if (ret > 0) {
2838 ret = btrfs_previous_extent_item(root, path, 0);
2839 if (ret < 0)
2840 goto out;
2841 if (ret > 0) {
2842 btrfs_release_path(path);
2843 ret = btrfs_search_slot(NULL, root, &key,
2844 path, 0, 0);
2845 if (ret < 0)
2846 goto out;
2847 }
2848 }
2849
2850 stop_loop = 0;
2851 while (1) {
2852 u64 bytes;
2853
2854 l = path->nodes[0];
2855 slot = path->slots[0];
2856 if (slot >= btrfs_header_nritems(l)) {
2857 ret = btrfs_next_leaf(root, path);
2858 if (ret == 0)
2859 continue;
2860 if (ret < 0)
2861 goto out;
2862
2863 stop_loop = 1;
2864 break;
2865 }
2866 btrfs_item_key_to_cpu(l, &key, slot);
2867
d7cad238
ZL		 2868 if (key.type != BTRFS_EXTENT_ITEM_KEY &&
2869 key.type != BTRFS_METADATA_ITEM_KEY)
2870 goto next;
2871
5a6ac9ea
MX		 2872 if (key.type == BTRFS_METADATA_ITEM_KEY)
2873 bytes = root->nodesize;
2874 else
2875 bytes = key.offset;
2876
2877 if (key.objectid + bytes <= logic_start)
2878 goto next;
2879
a0dd59de 2880 if (key.objectid >= logic_end) {
5a6ac9ea
MX		 2881 stop_loop = 1;
2882 break;
2883 }
2884
2885 while (key.objectid >= logic_start + map->stripe_len)
2886 logic_start += map->stripe_len;
2887
2888 extent = btrfs_item_ptr(l, slot,
2889 struct btrfs_extent_item);
2890 flags = btrfs_extent_flags(l, extent);
2891 generation = btrfs_extent_generation(l, extent);
2892
a323e813
ZL		 2893 if ((flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) &&
2894 (key.objectid < logic_start ||
2895 key.objectid + bytes >
2896 logic_start + map->stripe_len)) {
2897 btrfs_err(fs_info, "scrub: tree block %llu spanning stripes, ignored. logical=%llu",
2898 key.objectid, logic_start);
5a6ac9ea
MX		 2899 goto next;
2900 }
2901again:
2902 extent_logical = key.objectid;
2903 extent_len = bytes;
2904
2905 if (extent_logical < logic_start) {
2906 extent_len -= logic_start - extent_logical;
2907 extent_logical = logic_start;
2908 }
2909
2910 if (extent_logical + extent_len >
2911 logic_start + map->stripe_len)
2912 extent_len = logic_start + map->stripe_len -
2913 extent_logical;
2914
2915 scrub_parity_mark_sectors_data(sparity, extent_logical,
2916 extent_len);
2917
2918 scrub_remap_extent(fs_info, extent_logical,
2919 extent_len, &extent_physical,
2920 &extent_dev,
2921 &extent_mirror_num);
2922
2923 ret = btrfs_lookup_csums_range(csum_root,
2924 extent_logical,
2925 extent_logical + extent_len - 1,
2926 &sctx->csum_list, 1);
2927 if (ret)
2928 goto out;
2929
2930 ret = scrub_extent_for_parity(sparity, extent_logical,
2931 extent_len,
2932 extent_physical,
2933 extent_dev, flags,
2934 generation,
2935 extent_mirror_num);
6fa96d72
ZL		 2936
2937 scrub_free_csums(sctx);
2938
5a6ac9ea
MX		 2939 if (ret)
2940 goto out;
2941
5a6ac9ea
MX		 2942 if (extent_logical + extent_len <
2943 key.objectid + bytes) {
2944 logic_start += map->stripe_len;
2945
2946 if (logic_start >= logic_end) {
2947 stop_loop = 1;
2948 break;
2949 }
2950
2951 if (logic_start < key.objectid + bytes) {
2952 cond_resched();
2953 goto again;
2954 }
2955 }
2956next:
2957 path->slots[0]++;
2958 }
2959
2960 btrfs_release_path(path);
2961
2962 if (stop_loop)
2963 break;
2964
2965 logic_start += map->stripe_len;
2966 }
2967out:
2968 if (ret < 0)
2969 scrub_parity_mark_sectors_error(sparity, logic_start,
a0dd59de 2970 logic_end - logic_start);
5a6ac9ea
MX		 2971 scrub_parity_put(sparity);
2972 scrub_submit(sctx);
2973 mutex_lock(&sctx->wr_ctx.wr_lock);
2974 scrub_wr_submit(sctx);
2975 mutex_unlock(&sctx->wr_ctx.wr_lock);
2976
2977 btrfs_release_path(path);
2978 return ret < 0 ? ret : 0;
2979}
2980
d9d181c1 2981static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx,
a36cf8b8
SB		 2982 struct map_lookup *map,
2983 struct btrfs_device *scrub_dev,
ff023aac
SB		 2984 int num, u64 base, u64 length,
2985 int is_dev_replace)
a2de733c 2986{
5a6ac9ea 2987 struct btrfs_path *path, *ppath;
a36cf8b8 2988 struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
a2de733c
AJ		 2989 struct btrfs_root *root = fs_info->extent_root;
2990 struct btrfs_root *csum_root = fs_info->csum_root;
2991 struct btrfs_extent_item *extent;
e7786c3a 2992 struct blk_plug plug;
a2de733c
AJ		 2993 u64 flags;
2994 int ret;
2995 int slot;
a2de733c 2996 u64 nstripes;
a2de733c
AJ		 2997 struct extent_buffer *l;
2998 struct btrfs_key key;
2999 u64 physical;
3000 u64 logical;
625f1c8d 3001 u64 logic_end;
3b080b25 3002 u64 physical_end;
a2de733c 3003 u64 generation;
e12fa9cd 3004 int mirror_num;
7a26285e
AJ		 3005 struct reada_control *reada1;
3006 struct reada_control *reada2;
3007 struct btrfs_key key_start;
3008 struct btrfs_key key_end;
a2de733c
AJ		 3009 u64 increment = map->stripe_len;
3010 u64 offset;
ff023aac
SB		 3011 u64 extent_logical;
3012 u64 extent_physical;
3013 u64 extent_len;
5a6ac9ea
MX		 3014 u64 stripe_logical;
3015 u64 stripe_end;
ff023aac
SB		 3016 struct btrfs_device *extent_dev;
3017 int extent_mirror_num;
3b080b25 3018 int stop_loop = 0;
53b381b3 3019
3b080b25 3020 physical = map->stripes[num].physical;
a2de733c 3021 offset = 0;
b8b93add 3022 nstripes = div_u64(length, map->stripe_len);
a2de733c
AJ		 3023 if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
3024 offset = map->stripe_len * num;
3025 increment = map->stripe_len * map->num_stripes;
193ea74b 3026 mirror_num = 1;
a2de733c
AJ		 3027 } else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
3028 int factor = map->num_stripes / map->sub_stripes;
3029 offset = map->stripe_len * (num / map->sub_stripes);
3030 increment = map->stripe_len * factor;
193ea74b 3031 mirror_num = num % map->sub_stripes + 1;
a2de733c
AJ		 3032 } else if (map->type & BTRFS_BLOCK_GROUP_RAID1) {
3033 increment = map->stripe_len;
193ea74b 3034 mirror_num = num % map->num_stripes + 1;
a2de733c
AJ		 3035 } else if (map->type & BTRFS_BLOCK_GROUP_DUP) {
3036 increment = map->stripe_len;
193ea74b 3037 mirror_num = num % map->num_stripes + 1;
ffe2d203 3038 } else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
5a6ac9ea 3039 get_raid56_logic_offset(physical, num, map, &offset, NULL);
3b080b25
WS		 3040 increment = map->stripe_len * nr_data_stripes(map);
3041 mirror_num = 1;
a2de733c
AJ		 3042 } else {
3043 increment = map->stripe_len;
193ea74b 3044 mirror_num = 1;
a2de733c
AJ		 3045 }
3046
3047 path = btrfs_alloc_path();
3048 if (!path)
3049 return -ENOMEM;
3050
5a6ac9ea
MX		 3051 ppath = btrfs_alloc_path();
3052 if (!ppath) {
379d6854 3053 btrfs_free_path(path);
5a6ac9ea
MX		 3054 return -ENOMEM;
3055 }
3056
b5d67f64
SB		 3057 /*
3058 * work on commit root. The related disk blocks are static as
3059 * long as COW is applied. This means, it is save to rewrite
3060 * them to repair disk errors without any race conditions
3061 */
a2de733c
AJ		 3062 path->search_commit_root = 1;
3063 path->skip_locking = 1;
3064
063c54dc
GH		 3065 ppath->search_commit_root = 1;
3066 ppath->skip_locking = 1;
a2de733c 3067 /*
7a26285e
AJ		 3068 * trigger the readahead for extent tree csum tree and wait for
3069 * completion. During readahead, the scrub is officially paused
3070 * to not hold off transaction commits
a2de733c
AJ		 3071 */
3072 logical = base + offset;
3b080b25 3073 physical_end = physical + nstripes * map->stripe_len;
ffe2d203 3074 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3b080b25 3075 get_raid56_logic_offset(physical_end, num,
5a6ac9ea 3076 map, &logic_end, NULL);
3b080b25
WS		 3077 logic_end += base;
3078 } else {
3079 logic_end = logical + increment * nstripes;
3080 }
d9d181c1 3081 wait_event(sctx->list_wait,
b6bfebc1 3082 atomic_read(&sctx->bios_in_flight) == 0);
cb7ab021 3083 scrub_blocked_if_needed(fs_info);
7a26285e
AJ		 3084
3085 /* FIXME it might be better to start readahead at commit root */
3086 key_start.objectid = logical;
3087 key_start.type = BTRFS_EXTENT_ITEM_KEY;
3088 key_start.offset = (u64)0;
3b080b25 3089 key_end.objectid = logic_end;
3173a18f
JB		 3090 key_end.type = BTRFS_METADATA_ITEM_KEY;
3091 key_end.offset = (u64)-1;
7a26285e
AJ		 3092 reada1 = btrfs_reada_add(root, &key_start, &key_end);
3093
3094 key_start.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
3095 key_start.type = BTRFS_EXTENT_CSUM_KEY;
3096 key_start.offset = logical;
3097 key_end.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
3098 key_end.type = BTRFS_EXTENT_CSUM_KEY;
3b080b25 3099 key_end.offset = logic_end;
7a26285e
AJ		 3100 reada2 = btrfs_reada_add(csum_root, &key_start, &key_end);
3101
3102 if (!IS_ERR(reada1))
3103 btrfs_reada_wait(reada1);
3104 if (!IS_ERR(reada2))
3105 btrfs_reada_wait(reada2);
3106
a2de733c
AJ		 3107
3108 /*
3109 * collect all data csums for the stripe to avoid seeking during
3110 * the scrub. This might currently (crc32) end up to be about 1MB
3111 */
e7786c3a 3112 blk_start_plug(&plug);
a2de733c 3113
a2de733c
AJ		 3114 /*
3115 * now find all extents for each stripe and scrub them
3116 */
a2de733c 3117 ret = 0;
3b080b25 3118 while (physical < physical_end) {
a2de733c
AJ		 3119 /*
3120 * canceled?
3121 */
3122 if (atomic_read(&fs_info->scrub_cancel_req) ||
d9d181c1 3123 atomic_read(&sctx->cancel_req)) {
a2de733c
AJ		 3124 ret = -ECANCELED;
3125 goto out;
3126 }
3127 /*
3128 * check to see if we have to pause
3129 */
3130 if (atomic_read(&fs_info->scrub_pause_req)) {
3131 /* push queued extents */
ff023aac 3132 atomic_set(&sctx->wr_ctx.flush_all_writes, 1);
d9d181c1 3133 scrub_submit(sctx);
ff023aac
SB		 3134 mutex_lock(&sctx->wr_ctx.wr_lock);
3135 scrub_wr_submit(sctx);
3136 mutex_unlock(&sctx->wr_ctx.wr_lock);
d9d181c1 3137 wait_event(sctx->list_wait,
b6bfebc1 3138 atomic_read(&sctx->bios_in_flight) == 0);
ff023aac 3139 atomic_set(&sctx->wr_ctx.flush_all_writes, 0);
3cb0929a 3140 scrub_blocked_if_needed(fs_info);
a2de733c
AJ		 3141 }
3142
f2f66a2f
ZL		 3143 /* for raid56, we skip parity stripe */
3144 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3145 ret = get_raid56_logic_offset(physical, num, map,
3146 &logical,
3147 &stripe_logical);
3148 logical += base;
3149 if (ret) {
3150 stripe_logical += base;
a0dd59de 3151 stripe_end = stripe_logical + increment;
f2f66a2f
ZL		 3152 ret = scrub_raid56_parity(sctx, map, scrub_dev,
3153 ppath, stripe_logical,
3154 stripe_end);
3155 if (ret)
3156 goto out;
3157 goto skip;
3158 }
3159 }
3160
7c76edb7
WS		 3161 if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
3162 key.type = BTRFS_METADATA_ITEM_KEY;
3163 else
3164 key.type = BTRFS_EXTENT_ITEM_KEY;
a2de733c 3165 key.objectid = logical;
625f1c8d 3166 key.offset = (u64)-1;
a2de733c
AJ		 3167
3168 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3169 if (ret < 0)
3170 goto out;
3173a18f 3171
8c51032f 3172 if (ret > 0) {
ade2e0b3 3173 ret = btrfs_previous_extent_item(root, path, 0);
a2de733c
AJ		 3174 if (ret < 0)
3175 goto out;
8c51032f
AJ		 3176 if (ret > 0) {
3177 /* there's no smaller item, so stick with the
3178 * larger one */
3179 btrfs_release_path(path);
3180 ret = btrfs_search_slot(NULL, root, &key,
3181 path, 0, 0);
3182 if (ret < 0)
3183 goto out;
3184 }
a2de733c
AJ		 3185 }
3186
625f1c8d 3187 stop_loop = 0;
a2de733c 3188 while (1) {
3173a18f
JB		 3189 u64 bytes;
3190
a2de733c
AJ		 3191 l = path->nodes[0];
3192 slot = path->slots[0];
3193 if (slot >= btrfs_header_nritems(l)) {
3194 ret = btrfs_next_leaf(root, path);
3195 if (ret == 0)
3196 continue;
3197 if (ret < 0)
3198 goto out;
3199
625f1c8d 3200 stop_loop = 1;
a2de733c
AJ		 3201 break;
3202 }
3203 btrfs_item_key_to_cpu(l, &key, slot);
3204
d7cad238
ZL		 3205 if (key.type != BTRFS_EXTENT_ITEM_KEY &&
3206 key.type != BTRFS_METADATA_ITEM_KEY)
3207 goto next;
3208
3173a18f 3209 if (key.type == BTRFS_METADATA_ITEM_KEY)
707e8a07 3210 bytes = root->nodesize;
3173a18f
JB		 3211 else
3212 bytes = key.offset;
3213
3214 if (key.objectid + bytes <= logical)
a2de733c
AJ		 3215 goto next;
3216
625f1c8d
LB		 3217 if (key.objectid >= logical + map->stripe_len) {
3218 /* out of this device extent */
3219 if (key.objectid >= logic_end)
3220 stop_loop = 1;
3221 break;
3222 }
a2de733c
AJ		 3223
3224 extent = btrfs_item_ptr(l, slot,
3225 struct btrfs_extent_item);
3226 flags = btrfs_extent_flags(l, extent);
3227 generation = btrfs_extent_generation(l, extent);
3228
a323e813
ZL		 3229 if ((flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) &&
3230 (key.objectid < logical ||
3231 key.objectid + bytes >
3232 logical + map->stripe_len)) {
efe120a0
FH		 3233 btrfs_err(fs_info,
3234 "scrub: tree block %llu spanning "
3235 "stripes, ignored. logical=%llu",
c1c9ff7c 3236 key.objectid, logical);
a2de733c
AJ		 3237 goto next;
3238 }
3239
625f1c8d
LB		 3240again:
3241 extent_logical = key.objectid;
3242 extent_len = bytes;
3243
a2de733c
AJ		 3244 /*
3245 * trim extent to this stripe
3246 */
625f1c8d
LB		 3247 if (extent_logical < logical) {
3248 extent_len -= logical - extent_logical;
3249 extent_logical = logical;
a2de733c 3250 }
625f1c8d 3251 if (extent_logical + extent_len >
a2de733c 3252 logical + map->stripe_len) {
625f1c8d
LB		 3253 extent_len = logical + map->stripe_len -
3254 extent_logical;
a2de733c
AJ		 3255 }
3256
625f1c8d 3257 extent_physical = extent_logical - logical + physical;
ff023aac
SB		 3258 extent_dev = scrub_dev;
3259 extent_mirror_num = mirror_num;
3260 if (is_dev_replace)
3261 scrub_remap_extent(fs_info, extent_logical,
3262 extent_len, &extent_physical,
3263 &extent_dev,
3264 &extent_mirror_num);
625f1c8d 3265
fe8cf654
ZL		 3266 ret = btrfs_lookup_csums_range(csum_root,
3267 extent_logical,
3268 extent_logical +
3269 extent_len - 1,
3270 &sctx->csum_list, 1);
625f1c8d
LB		 3271 if (ret)
3272 goto out;
3273
ff023aac
SB		 3274 ret = scrub_extent(sctx, extent_logical, extent_len,
3275 extent_physical, extent_dev, flags,
3276 generation, extent_mirror_num,
115930cb 3277 extent_logical - logical + physical);
6fa96d72
ZL		 3278
3279 scrub_free_csums(sctx);
3280
a2de733c
AJ		 3281 if (ret)
3282 goto out;
3283
625f1c8d
LB		 3284 if (extent_logical + extent_len <
3285 key.objectid + bytes) {
ffe2d203 3286 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3b080b25
WS		 3287 /*
3288 * loop until we find next data stripe
3289 * or we have finished all stripes.
3290 */
5a6ac9ea
MX		 3291loop:
3292 physical += map->stripe_len;
3293 ret = get_raid56_logic_offset(physical,
3294 num, map, &logical,
3295 &stripe_logical);
3296 logical += base;
3297
3298 if (ret && physical < physical_end) {
3299 stripe_logical += base;
3300 stripe_end = stripe_logical +
a0dd59de 3301 increment;
5a6ac9ea
MX		 3302 ret = scrub_raid56_parity(sctx,
3303 map, scrub_dev, ppath,
3304 stripe_logical,
3305 stripe_end);
3306 if (ret)
3307 goto out;
3308 goto loop;
3309 }
3b080b25
WS		 3310 } else {
3311 physical += map->stripe_len;
3312 logical += increment;
3313 }
625f1c8d
LB		 3314 if (logical < key.objectid + bytes) {
3315 cond_resched();
3316 goto again;
3317 }
3318
3b080b25 3319 if (physical >= physical_end) {
625f1c8d
LB		 3320 stop_loop = 1;
3321 break;
3322 }
3323 }
a2de733c
AJ		 3324next:
3325 path->slots[0]++;
3326 }
71267333 3327 btrfs_release_path(path);
3b080b25 3328skip:
a2de733c
AJ		 3329 logical += increment;
3330 physical += map->stripe_len;
d9d181c1 3331 spin_lock(&sctx->stat_lock);
625f1c8d
LB		 3332 if (stop_loop)
3333 sctx->stat.last_physical = map->stripes[num].physical +
3334 length;
3335 else
3336 sctx->stat.last_physical = physical;
d9d181c1 3337 spin_unlock(&sctx->stat_lock);
625f1c8d
LB		 3338 if (stop_loop)
3339 break;
a2de733c 3340 }
ff023aac 3341out:
a2de733c 3342 /* push queued extents */
d9d181c1 3343 scrub_submit(sctx);
ff023aac
SB		 3344 mutex_lock(&sctx->wr_ctx.wr_lock);
3345 scrub_wr_submit(sctx);
3346 mutex_unlock(&sctx->wr_ctx.wr_lock);
a2de733c 3347
e7786c3a 3348 blk_finish_plug(&plug);
a2de733c 3349 btrfs_free_path(path);
5a6ac9ea 3350 btrfs_free_path(ppath);
a2de733c
AJ		 3351 return ret < 0 ? ret : 0;
3352}
3353
d9d181c1 3354static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx,
a36cf8b8
SB		 3355 struct btrfs_device *scrub_dev,
3356 u64 chunk_tree, u64 chunk_objectid,
3357 u64 chunk_offset, u64 length,
ff023aac 3358 u64 dev_offset, int is_dev_replace)
a2de733c
AJ		 3359{
3360 struct btrfs_mapping_tree *map_tree =
a36cf8b8 3361 &sctx->dev_root->fs_info->mapping_tree;
a2de733c
AJ		 3362 struct map_lookup *map;
3363 struct extent_map *em;
3364 int i;
ff023aac 3365 int ret = 0;
a2de733c
AJ		 3366
3367 read_lock(&map_tree->map_tree.lock);
3368 em = lookup_extent_mapping(&map_tree->map_tree, chunk_offset, 1);
3369 read_unlock(&map_tree->map_tree.lock);
3370
3371 if (!em)
3372 return -EINVAL;
3373
3374 map = (struct map_lookup *)em->bdev;
3375 if (em->start != chunk_offset)
3376 goto out;
3377
3378 if (em->len < length)
3379 goto out;
3380
3381 for (i = 0; i < map->num_stripes; ++i) {
a36cf8b8 3382 if (map->stripes[i].dev->bdev == scrub_dev->bdev &&
859acaf1 3383 map->stripes[i].physical == dev_offset) {
a36cf8b8 3384 ret = scrub_stripe(sctx, map, scrub_dev, i,
ff023aac
SB		 3385 chunk_offset, length,
3386 is_dev_replace);
a2de733c
AJ		 3387 if (ret)
3388 goto out;
3389 }
3390 }
3391out:
3392 free_extent_map(em);
3393
3394 return ret;
3395}
3396
3397static noinline_for_stack
a36cf8b8 3398int scrub_enumerate_chunks(struct scrub_ctx *sctx,
ff023aac
SB		 3399 struct btrfs_device *scrub_dev, u64 start, u64 end,
3400 int is_dev_replace)
a2de733c
AJ		 3401{
3402 struct btrfs_dev_extent *dev_extent = NULL;
3403 struct btrfs_path *path;
a36cf8b8 3404 struct btrfs_root *root = sctx->dev_root;
a2de733c
AJ		 3405 struct btrfs_fs_info *fs_info = root->fs_info;
3406 u64 length;
3407 u64 chunk_tree;
3408 u64 chunk_objectid;
3409 u64 chunk_offset;
55e3a601 3410 int ret = 0;
a2de733c
AJ		 3411 int slot;
3412 struct extent_buffer *l;
3413 struct btrfs_key key;
3414 struct btrfs_key found_key;
3415 struct btrfs_block_group_cache *cache;
ff023aac 3416 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
a2de733c
AJ		 3417
3418 path = btrfs_alloc_path();
3419 if (!path)
3420 return -ENOMEM;
3421
3422 path->reada = 2;
3423 path->search_commit_root = 1;
3424 path->skip_locking = 1;
3425
a36cf8b8 3426 key.objectid = scrub_dev->devid;
a2de733c
AJ		 3427 key.offset = 0ull;
3428 key.type = BTRFS_DEV_EXTENT_KEY;
3429
a2de733c
AJ		 3430 while (1) {
3431 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3432 if (ret < 0)
8c51032f
AJ		 3433 break;
3434 if (ret > 0) {
3435 if (path->slots[0] >=
3436 btrfs_header_nritems(path->nodes[0])) {
3437 ret = btrfs_next_leaf(root, path);
55e3a601
Z		 3438 if (ret < 0)
3439 break;
3440 if (ret > 0) {
3441 ret = 0;
8c51032f 3442 break;
55e3a601
Z		 3443 }
3444 } else {
3445 ret = 0;
8c51032f
AJ		 3446 }
3447 }
a2de733c
AJ		 3448
3449 l = path->nodes[0];
3450 slot = path->slots[0];
3451
3452 btrfs_item_key_to_cpu(l, &found_key, slot);
3453
a36cf8b8 3454 if (found_key.objectid != scrub_dev->devid)
a2de733c
AJ		 3455 break;
3456
962a298f 3457 if (found_key.type != BTRFS_DEV_EXTENT_KEY)
a2de733c
AJ		 3458 break;
3459
3460 if (found_key.offset >= end)
3461 break;
3462
3463 if (found_key.offset < key.offset)
3464 break;
3465
3466 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
3467 length = btrfs_dev_extent_length(l, dev_extent);
3468
ced96edc
QW		 3469 if (found_key.offset + length <= start)
3470 goto skip;
a2de733c
AJ		 3471
3472 chunk_tree = btrfs_dev_extent_chunk_tree(l, dev_extent);
3473 chunk_objectid = btrfs_dev_extent_chunk_objectid(l, dev_extent);
3474 chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);
3475
3476 /*
3477 * get a reference on the corresponding block group to prevent
3478 * the chunk from going away while we scrub it
3479 */
3480 cache = btrfs_lookup_block_group(fs_info, chunk_offset);
ced96edc
QW		 3481
3482 /* some chunks are removed but not committed to disk yet,
3483 * continue scrubbing */
3484 if (!cache)
3485 goto skip;
3486
55e3a601
Z		 3487 /*
3488 * we need call btrfs_inc_block_group_ro() with scrubs_paused,
3489 * to avoid deadlock caused by:
3490 * btrfs_inc_block_group_ro()
3491 * -> btrfs_wait_for_commit()
3492 * -> btrfs_commit_transaction()
3493 * -> btrfs_scrub_pause()
3494 */
3495 scrub_pause_on(fs_info);
3496 ret = btrfs_inc_block_group_ro(root, cache);
3497 scrub_pause_off(fs_info);
3498 if (ret) {
3499 btrfs_put_block_group(cache);
3500 break;
3501 }
3502
ff023aac
SB		 3503 dev_replace->cursor_right = found_key.offset + length;
3504 dev_replace->cursor_left = found_key.offset;
3505 dev_replace->item_needs_writeback = 1;
a36cf8b8 3506 ret = scrub_chunk(sctx, scrub_dev, chunk_tree, chunk_objectid,
ff023aac
SB		 3507 chunk_offset, length, found_key.offset,
3508 is_dev_replace);
3509
3510 /*
3511 * flush, submit all pending read and write bios, afterwards
3512 * wait for them.
3513 * Note that in the dev replace case, a read request causes
3514 * write requests that are submitted in the read completion
3515 * worker. Therefore in the current situation, it is required
3516 * that all write requests are flushed, so that all read and
3517 * write requests are really completed when bios_in_flight
3518 * changes to 0.
3519 */
3520 atomic_set(&sctx->wr_ctx.flush_all_writes, 1);
3521 scrub_submit(sctx);
3522 mutex_lock(&sctx->wr_ctx.wr_lock);
3523 scrub_wr_submit(sctx);
3524 mutex_unlock(&sctx->wr_ctx.wr_lock);
3525
3526 wait_event(sctx->list_wait,
3527 atomic_read(&sctx->bios_in_flight) == 0);
b708ce96
Z		 3528
3529 scrub_pause_on(fs_info);
12cf9372
WS		 3530
3531 /*
3532 * must be called before we decrease @scrub_paused.
3533 * make sure we don't block transaction commit while
3534 * we are waiting pending workers finished.
3535 */
ff023aac
SB		 3536 wait_event(sctx->list_wait,
3537 atomic_read(&sctx->workers_pending) == 0);
12cf9372
WS		 3538 atomic_set(&sctx->wr_ctx.flush_all_writes, 0);
3539
b708ce96 3540 scrub_pause_off(fs_info);
ff023aac 3541
55e3a601
Z		 3542 btrfs_dec_block_group_ro(root, cache);
3543
a2de733c
AJ		 3544 btrfs_put_block_group(cache);
3545 if (ret)
3546 break;
af1be4f8
SB		 3547 if (is_dev_replace &&
3548 atomic64_read(&dev_replace->num_write_errors) > 0) {
ff023aac
SB		 3549 ret = -EIO;
3550 break;
3551 }
3552 if (sctx->stat.malloc_errors > 0) {
3553 ret = -ENOMEM;
3554 break;
3555 }
a2de733c 3556
539f358a
ID		 3557 dev_replace->cursor_left = dev_replace->cursor_right;
3558 dev_replace->item_needs_writeback = 1;
ced96edc 3559skip:
a2de733c 3560 key.offset = found_key.offset + length;
71267333 3561 btrfs_release_path(path);
a2de733c
AJ		 3562 }
3563
a2de733c 3564 btrfs_free_path(path);
8c51032f 3565
55e3a601 3566 return ret;
a2de733c
AJ		 3567}
3568
a36cf8b8
SB		 3569static noinline_for_stack int scrub_supers(struct scrub_ctx *sctx,
3570 struct btrfs_device *scrub_dev)
a2de733c
AJ		 3571{
3572 int i;
3573 u64 bytenr;
3574 u64 gen;
3575 int ret;
a36cf8b8 3576 struct btrfs_root *root = sctx->dev_root;
a2de733c 3577
87533c47 3578 if (test_bit(BTRFS_FS_STATE_ERROR, &root->fs_info->fs_state))
79787eaa
JM		 3579 return -EIO;
3580
5f546063
MX		 3581 /* Seed devices of a new filesystem has their own generation. */
3582 if (scrub_dev->fs_devices != root->fs_info->fs_devices)
3583 gen = scrub_dev->generation;
3584 else
3585 gen = root->fs_info->last_trans_committed;
a2de733c
AJ		 3586
3587 for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
3588 bytenr = btrfs_sb_offset(i);
935e5cc9
MX		 3589 if (bytenr + BTRFS_SUPER_INFO_SIZE >
3590 scrub_dev->commit_total_bytes)
a2de733c
AJ		 3591 break;
3592
d9d181c1 3593 ret = scrub_pages(sctx, bytenr, BTRFS_SUPER_INFO_SIZE, bytenr,
a36cf8b8 3594 scrub_dev, BTRFS_EXTENT_FLAG_SUPER, gen, i,
ff023aac 3595 NULL, 1, bytenr);
a2de733c
AJ		 3596 if (ret)
3597 return ret;
3598 }
b6bfebc1 3599 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
a2de733c
AJ		 3600
3601 return 0;
3602}
3603
3604/*
3605 * get a reference count on fs_info->scrub_workers. start worker if necessary
3606 */
ff023aac
SB		 3607static noinline_for_stack int scrub_workers_get(struct btrfs_fs_info *fs_info,
3608 int is_dev_replace)
a2de733c 3609{
6f011058 3610 unsigned int flags = WQ_FREEZABLE | WQ_UNBOUND;
0339ef2f 3611 int max_active = fs_info->thread_pool_size;
a2de733c 3612
632dd772 3613 if (fs_info->scrub_workers_refcnt == 0) {
ff023aac 3614 if (is_dev_replace)
0339ef2f
QW		 3615 fs_info->scrub_workers =
3616 btrfs_alloc_workqueue("btrfs-scrub", flags,
3617 1, 4);
ff023aac 3618 else
0339ef2f
QW		 3619 fs_info->scrub_workers =
3620 btrfs_alloc_workqueue("btrfs-scrub", flags,
3621 max_active, 4);
e82afc52
ZL		 3622 if (!fs_info->scrub_workers)
3623 goto fail_scrub_workers;
3624
0339ef2f
QW		 3625 fs_info->scrub_wr_completion_workers =
3626 btrfs_alloc_workqueue("btrfs-scrubwrc", flags,
3627 max_active, 2);
e82afc52
ZL		 3628 if (!fs_info->scrub_wr_completion_workers)
3629 goto fail_scrub_wr_completion_workers;
3630
0339ef2f
QW		 3631 fs_info->scrub_nocow_workers =
3632 btrfs_alloc_workqueue("btrfs-scrubnc", flags, 1, 0);
e82afc52
ZL		 3633 if (!fs_info->scrub_nocow_workers)
3634 goto fail_scrub_nocow_workers;
20b2e302
ZL		 3635 fs_info->scrub_parity_workers =
3636 btrfs_alloc_workqueue("btrfs-scrubparity", flags,
3637 max_active, 2);
e82afc52
ZL		 3638 if (!fs_info->scrub_parity_workers)
3639 goto fail_scrub_parity_workers;
632dd772 3640 }
a2de733c 3641 ++fs_info->scrub_workers_refcnt;
e82afc52
ZL		 3642 return 0;
3643
3644fail_scrub_parity_workers:
3645 btrfs_destroy_workqueue(fs_info->scrub_nocow_workers);
3646fail_scrub_nocow_workers:
3647 btrfs_destroy_workqueue(fs_info->scrub_wr_completion_workers);
3648fail_scrub_wr_completion_workers:
3649 btrfs_destroy_workqueue(fs_info->scrub_workers);
3650fail_scrub_workers:
3651 return -ENOMEM;
a2de733c
AJ		 3652}
3653
aa1b8cd4 3654static noinline_for_stack void scrub_workers_put(struct btrfs_fs_info *fs_info)
a2de733c 3655{
ff023aac 3656 if (--fs_info->scrub_workers_refcnt == 0) {
0339ef2f
QW		 3657 btrfs_destroy_workqueue(fs_info->scrub_workers);
3658 btrfs_destroy_workqueue(fs_info->scrub_wr_completion_workers);
3659 btrfs_destroy_workqueue(fs_info->scrub_nocow_workers);
20b2e302 3660 btrfs_destroy_workqueue(fs_info->scrub_parity_workers);
ff023aac 3661 }
a2de733c 3662 WARN_ON(fs_info->scrub_workers_refcnt < 0);
a2de733c
AJ		 3663}
3664
aa1b8cd4
SB		 3665int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start,
3666 u64 end, struct btrfs_scrub_progress *progress,
63a212ab 3667 int readonly, int is_dev_replace)
a2de733c 3668{
d9d181c1 3669 struct scrub_ctx *sctx;
a2de733c
AJ		 3670 int ret;
3671 struct btrfs_device *dev;
5d68da3b 3672 struct rcu_string *name;
a2de733c 3673
aa1b8cd4 3674 if (btrfs_fs_closing(fs_info))
a2de733c
AJ		 3675 return -EINVAL;
3676
aa1b8cd4 3677 if (fs_info->chunk_root->nodesize > BTRFS_STRIPE_LEN) {
b5d67f64
SB		 3678 /*
3679 * in this case scrub is unable to calculate the checksum
3680 * the way scrub is implemented. Do not handle this
3681 * situation at all because it won't ever happen.
3682 */
efe120a0
FH		 3683 btrfs_err(fs_info,
3684 "scrub: size assumption nodesize <= BTRFS_STRIPE_LEN (%d <= %d) fails",
aa1b8cd4 3685 fs_info->chunk_root->nodesize, BTRFS_STRIPE_LEN);
b5d67f64
SB		 3686 return -EINVAL;
3687 }
3688
aa1b8cd4 3689 if (fs_info->chunk_root->sectorsize != PAGE_SIZE) {
b5d67f64 3690 /* not supported for data w/o checksums */
efe120a0
FH		 3691 btrfs_err(fs_info,
3692 "scrub: size assumption sectorsize != PAGE_SIZE "
3693 "(%d != %lu) fails",
27f9f023 3694 fs_info->chunk_root->sectorsize, PAGE_SIZE);
a2de733c
AJ		 3695 return -EINVAL;
3696 }
3697
7a9e9987
SB		 3698 if (fs_info->chunk_root->nodesize >
3699 PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK ||
3700 fs_info->chunk_root->sectorsize >
3701 PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK) {
3702 /*
3703 * would exhaust the array bounds of pagev member in
3704 * struct scrub_block
3705 */
efe120a0
FH		 3706 btrfs_err(fs_info, "scrub: size assumption nodesize and sectorsize "
3707 "<= SCRUB_MAX_PAGES_PER_BLOCK (%d <= %d && %d <= %d) fails",
7a9e9987
SB		 3708 fs_info->chunk_root->nodesize,
3709 SCRUB_MAX_PAGES_PER_BLOCK,
3710 fs_info->chunk_root->sectorsize,
3711 SCRUB_MAX_PAGES_PER_BLOCK);
3712 return -EINVAL;
3713 }
3714
a2de733c 3715
aa1b8cd4
SB		 3716 mutex_lock(&fs_info->fs_devices->device_list_mutex);
3717 dev = btrfs_find_device(fs_info, devid, NULL, NULL);
63a212ab 3718 if (!dev || (dev->missing && !is_dev_replace)) {
aa1b8cd4 3719 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
a2de733c
AJ		 3720 return -ENODEV;
3721 }
a2de733c 3722
5d68da3b
MX		 3723 if (!is_dev_replace && !readonly && !dev->writeable) {
3724 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3725 rcu_read_lock();
3726 name = rcu_dereference(dev->name);
3727 btrfs_err(fs_info, "scrub: device %s is not writable",
3728 name->str);
3729 rcu_read_unlock();
3730 return -EROFS;
3731 }
3732
3b7a016f 3733 mutex_lock(&fs_info->scrub_lock);
63a212ab 3734 if (!dev->in_fs_metadata || dev->is_tgtdev_for_dev_replace) {
a2de733c 3735 mutex_unlock(&fs_info->scrub_lock);
aa1b8cd4 3736 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
aa1b8cd4 3737 return -EIO;
a2de733c
AJ		 3738 }
3739
8dabb742
SB		 3740 btrfs_dev_replace_lock(&fs_info->dev_replace);
3741 if (dev->scrub_device ||
3742 (!is_dev_replace &&
3743 btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))) {
3744 btrfs_dev_replace_unlock(&fs_info->dev_replace);
a2de733c 3745 mutex_unlock(&fs_info->scrub_lock);
aa1b8cd4 3746 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
a2de733c
AJ		 3747 return -EINPROGRESS;
3748 }
8dabb742 3749 btrfs_dev_replace_unlock(&fs_info->dev_replace);
3b7a016f
WS		 3750
3751 ret = scrub_workers_get(fs_info, is_dev_replace);
3752 if (ret) {
3753 mutex_unlock(&fs_info->scrub_lock);
3754 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3755 return ret;
3756 }
3757
63a212ab 3758 sctx = scrub_setup_ctx(dev, is_dev_replace);
d9d181c1 3759 if (IS_ERR(sctx)) {
a2de733c 3760 mutex_unlock(&fs_info->scrub_lock);
aa1b8cd4
SB		 3761 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3762 scrub_workers_put(fs_info);
d9d181c1 3763 return PTR_ERR(sctx);
a2de733c 3764 }
d9d181c1
SB		 3765 sctx->readonly = readonly;
3766 dev->scrub_device = sctx;
3cb0929a 3767 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
a2de733c 3768
3cb0929a
WS		 3769 /*
3770 * checking @scrub_pause_req here, we can avoid
3771 * race between committing transaction and scrubbing.
3772 */
cb7ab021 3773 __scrub_blocked_if_needed(fs_info);
a2de733c
AJ		 3774 atomic_inc(&fs_info->scrubs_running);
3775 mutex_unlock(&fs_info->scrub_lock);
a2de733c 3776
ff023aac 3777 if (!is_dev_replace) {
9b011adf
WS		 3778 /*
3779 * by holding device list mutex, we can
3780 * kick off writing super in log tree sync.
3781 */
3cb0929a 3782 mutex_lock(&fs_info->fs_devices->device_list_mutex);
ff023aac 3783 ret = scrub_supers(sctx, dev);
3cb0929a 3784 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
ff023aac 3785 }
a2de733c
AJ		 3786
3787 if (!ret)
ff023aac
SB		 3788 ret = scrub_enumerate_chunks(sctx, dev, start, end,
3789 is_dev_replace);
a2de733c 3790
b6bfebc1 3791 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
a2de733c
AJ		 3792 atomic_dec(&fs_info->scrubs_running);
3793 wake_up(&fs_info->scrub_pause_wait);
3794
b6bfebc1 3795 wait_event(sctx->list_wait, atomic_read(&sctx->workers_pending) == 0);
0ef8e451 3796
a2de733c 3797 if (progress)
d9d181c1 3798 memcpy(progress, &sctx->stat, sizeof(*progress));
a2de733c
AJ		 3799
3800 mutex_lock(&fs_info->scrub_lock);
3801 dev->scrub_device = NULL;
3b7a016f 3802 scrub_workers_put(fs_info);
a2de733c
AJ		 3803 mutex_unlock(&fs_info->scrub_lock);
3804
f55985f4 3805 scrub_put_ctx(sctx);
a2de733c
AJ		 3806
3807 return ret;
3808}
3809
143bede5 3810void btrfs_scrub_pause(struct btrfs_root *root)
a2de733c
AJ		 3811{
3812 struct btrfs_fs_info *fs_info = root->fs_info;
3813
3814 mutex_lock(&fs_info->scrub_lock);
3815 atomic_inc(&fs_info->scrub_pause_req);
3816 while (atomic_read(&fs_info->scrubs_paused) !=
3817 atomic_read(&fs_info->scrubs_running)) {
3818 mutex_unlock(&fs_info->scrub_lock);
3819 wait_event(fs_info->scrub_pause_wait,
3820 atomic_read(&fs_info->scrubs_paused) ==
3821 atomic_read(&fs_info->scrubs_running));
3822 mutex_lock(&fs_info->scrub_lock);
3823 }
3824 mutex_unlock(&fs_info->scrub_lock);
a2de733c
AJ		 3825}
3826
143bede5 3827void btrfs_scrub_continue(struct btrfs_root *root)
a2de733c
AJ		 3828{
3829 struct btrfs_fs_info *fs_info = root->fs_info;
3830
3831 atomic_dec(&fs_info->scrub_pause_req);
3832 wake_up(&fs_info->scrub_pause_wait);
a2de733c
AJ		 3833}
3834
aa1b8cd4 3835int btrfs_scrub_cancel(struct btrfs_fs_info *fs_info)
a2de733c 3836{
a2de733c
AJ		 3837 mutex_lock(&fs_info->scrub_lock);
3838 if (!atomic_read(&fs_info->scrubs_running)) {
3839 mutex_unlock(&fs_info->scrub_lock);
3840 return -ENOTCONN;
3841 }
3842
3843 atomic_inc(&fs_info->scrub_cancel_req);
3844 while (atomic_read(&fs_info->scrubs_running)) {
3845 mutex_unlock(&fs_info->scrub_lock);
3846 wait_event(fs_info->scrub_pause_wait,
3847 atomic_read(&fs_info->scrubs_running) == 0);
3848 mutex_lock(&fs_info->scrub_lock);
3849 }
3850 atomic_dec(&fs_info->scrub_cancel_req);
3851 mutex_unlock(&fs_info->scrub_lock);
3852
3853 return 0;
3854}
3855
aa1b8cd4
SB		 3856int btrfs_scrub_cancel_dev(struct btrfs_fs_info *fs_info,
3857 struct btrfs_device *dev)
49b25e05 3858{
d9d181c1 3859 struct scrub_ctx *sctx;
a2de733c
AJ		 3860
3861 mutex_lock(&fs_info->scrub_lock);
d9d181c1
SB		 3862 sctx = dev->scrub_device;
3863 if (!sctx) {
a2de733c
AJ		 3864 mutex_unlock(&fs_info->scrub_lock);
3865 return -ENOTCONN;
3866 }
d9d181c1 3867 atomic_inc(&sctx->cancel_req);
a2de733c
AJ		 3868 while (dev->scrub_device) {
3869 mutex_unlock(&fs_info->scrub_lock);
3870 wait_event(fs_info->scrub_pause_wait,
3871 dev->scrub_device == NULL);
3872 mutex_lock(&fs_info->scrub_lock);
3873 }
3874 mutex_unlock(&fs_info->scrub_lock);
3875
3876 return 0;
3877}
1623edeb 3878
a2de733c
AJ		 3879int btrfs_scrub_progress(struct btrfs_root *root, u64 devid,
3880 struct btrfs_scrub_progress *progress)
3881{
3882 struct btrfs_device *dev;
d9d181c1 3883 struct scrub_ctx *sctx = NULL;
a2de733c
AJ		 3884
3885 mutex_lock(&root->fs_info->fs_devices->device_list_mutex);
aa1b8cd4 3886 dev = btrfs_find_device(root->fs_info, devid, NULL, NULL);
a2de733c 3887 if (dev)
d9d181c1
SB		 3888 sctx = dev->scrub_device;
3889 if (sctx)
3890 memcpy(progress, &sctx->stat, sizeof(*progress));
a2de733c
AJ		 3891 mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
3892
d9d181c1 3893 return dev ? (sctx ? 0 : -ENOTCONN) : -ENODEV;
a2de733c 3894}
ff023aac
SB		 3895
3896static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
3897 u64 extent_logical, u64 extent_len,
3898 u64 *extent_physical,
3899 struct btrfs_device **extent_dev,
3900 int *extent_mirror_num)
3901{
3902 u64 mapped_length;
3903 struct btrfs_bio *bbio = NULL;
3904 int ret;
3905
3906 mapped_length = extent_len;
3907 ret = btrfs_map_block(fs_info, READ, extent_logical,
3908 &mapped_length, &bbio, 0);
3909 if (ret || !bbio || mapped_length < extent_len ||
3910 !bbio->stripes[0].dev->bdev) {
6e9606d2 3911 btrfs_put_bbio(bbio);
ff023aac
SB		 3912 return;
3913 }
3914
3915 *extent_physical = bbio->stripes[0].physical;
3916 *extent_mirror_num = bbio->mirror_num;
3917 *extent_dev = bbio->stripes[0].dev;
6e9606d2 3918 btrfs_put_bbio(bbio);
ff023aac
SB		 3919}
3920
3921static int scrub_setup_wr_ctx(struct scrub_ctx *sctx,
3922 struct scrub_wr_ctx *wr_ctx,
3923 struct btrfs_fs_info *fs_info,
3924 struct btrfs_device *dev,
3925 int is_dev_replace)
3926{
3927 WARN_ON(wr_ctx->wr_curr_bio != NULL);
3928
3929 mutex_init(&wr_ctx->wr_lock);
3930 wr_ctx->wr_curr_bio = NULL;
3931 if (!is_dev_replace)
3932 return 0;
3933
3934 WARN_ON(!dev->bdev);
3935 wr_ctx->pages_per_wr_bio = min_t(int, SCRUB_PAGES_PER_WR_BIO,
3936 bio_get_nr_vecs(dev->bdev));
3937 wr_ctx->tgtdev = dev;
3938 atomic_set(&wr_ctx->flush_all_writes, 0);
3939 return 0;
3940}
3941
3942static void scrub_free_wr_ctx(struct scrub_wr_ctx *wr_ctx)
3943{
3944 mutex_lock(&wr_ctx->wr_lock);
3945 kfree(wr_ctx->wr_curr_bio);
3946 wr_ctx->wr_curr_bio = NULL;
3947 mutex_unlock(&wr_ctx->wr_lock);
3948}
3949
3950static int copy_nocow_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
3951 int mirror_num, u64 physical_for_dev_replace)
3952{
3953 struct scrub_copy_nocow_ctx *nocow_ctx;
3954 struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
3955
3956 nocow_ctx = kzalloc(sizeof(*nocow_ctx), GFP_NOFS);
3957 if (!nocow_ctx) {
3958 spin_lock(&sctx->stat_lock);
3959 sctx->stat.malloc_errors++;
3960 spin_unlock(&sctx->stat_lock);
3961 return -ENOMEM;
3962 }
3963
3964 scrub_pending_trans_workers_inc(sctx);
3965
3966 nocow_ctx->sctx = sctx;
3967 nocow_ctx->logical = logical;
3968 nocow_ctx->len = len;
3969 nocow_ctx->mirror_num = mirror_num;
3970 nocow_ctx->physical_for_dev_replace = physical_for_dev_replace;
9e0af237
LB		 3971 btrfs_init_work(&nocow_ctx->work, btrfs_scrubnc_helper,
3972 copy_nocow_pages_worker, NULL, NULL);
652f25a2 3973 INIT_LIST_HEAD(&nocow_ctx->inodes);
0339ef2f
QW		 3974 btrfs_queue_work(fs_info->scrub_nocow_workers,
3975 &nocow_ctx->work);
ff023aac
SB		 3976
3977 return 0;
3978}
3979
652f25a2
JB		 3980static int record_inode_for_nocow(u64 inum, u64 offset, u64 root, void *ctx)
3981{
3982 struct scrub_copy_nocow_ctx *nocow_ctx = ctx;
3983 struct scrub_nocow_inode *nocow_inode;
3984
3985 nocow_inode = kzalloc(sizeof(*nocow_inode), GFP_NOFS);
3986 if (!nocow_inode)
3987 return -ENOMEM;
3988 nocow_inode->inum = inum;
3989 nocow_inode->offset = offset;
3990 nocow_inode->root = root;
3991 list_add_tail(&nocow_inode->list, &nocow_ctx->inodes);
3992 return 0;
3993}
3994
3995#define COPY_COMPLETE 1
3996
ff023aac
SB		 3997static void copy_nocow_pages_worker(struct btrfs_work *work)
3998{
3999 struct scrub_copy_nocow_ctx *nocow_ctx =
4000 container_of(work, struct scrub_copy_nocow_ctx, work);
4001 struct scrub_ctx *sctx = nocow_ctx->sctx;
4002 u64 logical = nocow_ctx->logical;
4003 u64 len = nocow_ctx->len;
4004 int mirror_num = nocow_ctx->mirror_num;
4005 u64 physical_for_dev_replace = nocow_ctx->physical_for_dev_replace;
4006 int ret;
4007 struct btrfs_trans_handle *trans = NULL;
4008 struct btrfs_fs_info *fs_info;
4009 struct btrfs_path *path;
4010 struct btrfs_root *root;
4011 int not_written = 0;
4012
4013 fs_info = sctx->dev_root->fs_info;
4014 root = fs_info->extent_root;
4015
4016 path = btrfs_alloc_path();
4017 if (!path) {
4018 spin_lock(&sctx->stat_lock);
4019 sctx->stat.malloc_errors++;
4020 spin_unlock(&sctx->stat_lock);
4021 not_written = 1;
4022 goto out;
4023 }
4024
4025 trans = btrfs_join_transaction(root);
4026 if (IS_ERR(trans)) {
4027 not_written = 1;
4028 goto out;
4029 }
4030
4031 ret = iterate_inodes_from_logical(logical, fs_info, path,
652f25a2 4032 record_inode_for_nocow, nocow_ctx);
ff023aac 4033 if (ret != 0 && ret != -ENOENT) {
efe120a0
FH		 4034 btrfs_warn(fs_info, "iterate_inodes_from_logical() failed: log %llu, "
4035 "phys %llu, len %llu, mir %u, ret %d",
118a0a25
GU		 4036 logical, physical_for_dev_replace, len, mirror_num,
4037 ret);
ff023aac
SB		 4038 not_written = 1;
4039 goto out;
4040 }
4041
652f25a2
JB		 4042 btrfs_end_transaction(trans, root);
4043 trans = NULL;
4044 while (!list_empty(&nocow_ctx->inodes)) {
4045 struct scrub_nocow_inode *entry;
4046 entry = list_first_entry(&nocow_ctx->inodes,
4047 struct scrub_nocow_inode,
4048 list);
4049 list_del_init(&entry->list);
4050 ret = copy_nocow_pages_for_inode(entry->inum, entry->offset,
4051 entry->root, nocow_ctx);
4052 kfree(entry);
4053 if (ret == COPY_COMPLETE) {
4054 ret = 0;
4055 break;
4056 } else if (ret) {
4057 break;
4058 }
4059 }
ff023aac 4060out:
652f25a2
JB		 4061 while (!list_empty(&nocow_ctx->inodes)) {
4062 struct scrub_nocow_inode *entry;
4063 entry = list_first_entry(&nocow_ctx->inodes,
4064 struct scrub_nocow_inode,
4065 list);
4066 list_del_init(&entry->list);
4067 kfree(entry);
4068 }
ff023aac
SB		 4069 if (trans && !IS_ERR(trans))
4070 btrfs_end_transaction(trans, root);
4071 if (not_written)
4072 btrfs_dev_replace_stats_inc(&fs_info->dev_replace.
4073 num_uncorrectable_read_errors);
4074
4075 btrfs_free_path(path);
4076 kfree(nocow_ctx);
4077
4078 scrub_pending_trans_workers_dec(sctx);
4079}
4080
32159242
GH		 4081static int check_extent_to_block(struct inode *inode, u64 start, u64 len,
4082 u64 logical)
4083{
4084 struct extent_state *cached_state = NULL;
4085 struct btrfs_ordered_extent *ordered;
4086 struct extent_io_tree *io_tree;
4087 struct extent_map *em;
4088 u64 lockstart = start, lockend = start + len - 1;
4089 int ret = 0;
4090
4091 io_tree = &BTRFS_I(inode)->io_tree;
4092
4093 lock_extent_bits(io_tree, lockstart, lockend, 0, &cached_state);
4094 ordered = btrfs_lookup_ordered_range(inode, lockstart, len);
4095 if (ordered) {
4096 btrfs_put_ordered_extent(ordered);
4097 ret = 1;
4098 goto out_unlock;
4099 }
4100
4101 em = btrfs_get_extent(inode, NULL, 0, start, len, 0);
4102 if (IS_ERR(em)) {
4103 ret = PTR_ERR(em);
4104 goto out_unlock;
4105 }
4106
4107 /*
4108 * This extent does not actually cover the logical extent anymore,
4109 * move on to the next inode.
4110 */
4111 if (em->block_start > logical ||
4112 em->block_start + em->block_len < logical + len) {
4113 free_extent_map(em);
4114 ret = 1;
4115 goto out_unlock;
4116 }
4117 free_extent_map(em);
4118
4119out_unlock:
4120 unlock_extent_cached(io_tree, lockstart, lockend, &cached_state,
4121 GFP_NOFS);
4122 return ret;
4123}
4124
652f25a2
JB		 4125static int copy_nocow_pages_for_inode(u64 inum, u64 offset, u64 root,
4126 struct scrub_copy_nocow_ctx *nocow_ctx)
ff023aac 4127{
826aa0a8 4128 struct btrfs_fs_info *fs_info = nocow_ctx->sctx->dev_root->fs_info;
ff023aac 4129 struct btrfs_key key;
826aa0a8
MX		 4130 struct inode *inode;
4131 struct page *page;
ff023aac 4132 struct btrfs_root *local_root;
652f25a2 4133 struct extent_io_tree *io_tree;
ff023aac 4134 u64 physical_for_dev_replace;
32159242 4135 u64 nocow_ctx_logical;
652f25a2 4136 u64 len = nocow_ctx->len;
826aa0a8 4137 unsigned long index;
6f1c3605 4138 int srcu_index;
652f25a2
JB		 4139 int ret = 0;
4140 int err = 0;
ff023aac
SB		 4141
4142 key.objectid = root;
4143 key.type = BTRFS_ROOT_ITEM_KEY;
4144 key.offset = (u64)-1;
6f1c3605
LB		 4145
4146 srcu_index = srcu_read_lock(&fs_info->subvol_srcu);
4147
ff023aac 4148 local_root = btrfs_read_fs_root_no_name(fs_info, &key);
6f1c3605
LB		 4149 if (IS_ERR(local_root)) {
4150 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
ff023aac 4151 return PTR_ERR(local_root);
6f1c3605 4152 }
ff023aac
SB		 4153
4154 key.type = BTRFS_INODE_ITEM_KEY;
4155 key.objectid = inum;
4156 key.offset = 0;
4157 inode = btrfs_iget(fs_info->sb, &key, local_root, NULL);
6f1c3605 4158 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
ff023aac
SB		 4159 if (IS_ERR(inode))
4160 return PTR_ERR(inode);
4161
edd1400b
MX		 4162 /* Avoid truncate/dio/punch hole.. */
4163 mutex_lock(&inode->i_mutex);
4164 inode_dio_wait(inode);
4165
ff023aac 4166 physical_for_dev_replace = nocow_ctx->physical_for_dev_replace;
652f25a2 4167 io_tree = &BTRFS_I(inode)->io_tree;
32159242 4168 nocow_ctx_logical = nocow_ctx->logical;
652f25a2 4169
32159242
GH		 4170 ret = check_extent_to_block(inode, offset, len, nocow_ctx_logical);
4171 if (ret) {
4172 ret = ret > 0 ? 0 : ret;
4173 goto out;
652f25a2 4174 }
652f25a2 4175
ff023aac 4176 while (len >= PAGE_CACHE_SIZE) {
ff023aac 4177 index = offset >> PAGE_CACHE_SHIFT;
edd1400b 4178again:
ff023aac
SB		 4179 page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
4180 if (!page) {
efe120a0 4181 btrfs_err(fs_info, "find_or_create_page() failed");
ff023aac 4182 ret = -ENOMEM;
826aa0a8 4183 goto out;
ff023aac
SB		 4184 }
4185
4186 if (PageUptodate(page)) {
4187 if (PageDirty(page))
4188 goto next_page;
4189 } else {
4190 ClearPageError(page);
32159242 4191 err = extent_read_full_page(io_tree, page,
652f25a2
JB		 4192 btrfs_get_extent,
4193 nocow_ctx->mirror_num);
826aa0a8
MX		 4194 if (err) {
4195 ret = err;
ff023aac
SB		 4196 goto next_page;
4197 }
edd1400b 4198
26b25891 4199 lock_page(page);
edd1400b
MX		 4200 /*
4201 * If the page has been remove from the page cache,
4202 * the data on it is meaningless, because it may be
4203 * old one, the new data may be written into the new
4204 * page in the page cache.
4205 */
4206 if (page->mapping != inode->i_mapping) {
652f25a2 4207 unlock_page(page);
edd1400b
MX		 4208 page_cache_release(page);
4209 goto again;
4210 }
ff023aac
SB		 4211 if (!PageUptodate(page)) {
4212 ret = -EIO;
4213 goto next_page;
4214 }
4215 }
32159242
GH		 4216
4217 ret = check_extent_to_block(inode, offset, len,
4218 nocow_ctx_logical);
4219 if (ret) {
4220 ret = ret > 0 ? 0 : ret;
4221 goto next_page;
4222 }
4223
826aa0a8
MX		 4224 err = write_page_nocow(nocow_ctx->sctx,
4225 physical_for_dev_replace, page);
4226 if (err)
4227 ret = err;
ff023aac 4228next_page:
826aa0a8
MX		 4229 unlock_page(page);
4230 page_cache_release(page);
4231
4232 if (ret)
4233 break;
4234
ff023aac
SB		 4235 offset += PAGE_CACHE_SIZE;
4236 physical_for_dev_replace += PAGE_CACHE_SIZE;
32159242 4237 nocow_ctx_logical += PAGE_CACHE_SIZE;
ff023aac
SB		 4238 len -= PAGE_CACHE_SIZE;
4239 }
652f25a2 4240 ret = COPY_COMPLETE;
826aa0a8 4241out:
edd1400b 4242 mutex_unlock(&inode->i_mutex);
826aa0a8 4243 iput(inode);
ff023aac
SB		 4244 return ret;
4245}
4246
4247static int write_page_nocow(struct scrub_ctx *sctx,
4248 u64 physical_for_dev_replace, struct page *page)
4249{
4250 struct bio *bio;
4251 struct btrfs_device *dev;
4252 int ret;
ff023aac
SB		 4253
4254 dev = sctx->wr_ctx.tgtdev;
4255 if (!dev)
4256 return -EIO;
4257 if (!dev->bdev) {
4258 printk_ratelimited(KERN_WARNING
efe120a0 4259 "BTRFS: scrub write_page_nocow(bdev == NULL) is unexpected!\n");
ff023aac
SB		 4260 return -EIO;
4261 }
9be3395b 4262 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
ff023aac
SB		 4263 if (!bio) {
4264 spin_lock(&sctx->stat_lock);
4265 sctx->stat.malloc_errors++;
4266 spin_unlock(&sctx->stat_lock);
4267 return -ENOMEM;
4268 }
4f024f37
KO		 4269 bio->bi_iter.bi_size = 0;
4270 bio->bi_iter.bi_sector = physical_for_dev_replace >> 9;
ff023aac
SB		 4271 bio->bi_bdev = dev->bdev;
4272 ret = bio_add_page(bio, page, PAGE_CACHE_SIZE, 0);
4273 if (ret != PAGE_CACHE_SIZE) {
4274leave_with_eio:
4275 bio_put(bio);
4276 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
4277 return -EIO;
4278 }
ff023aac 4279
33879d45 4280 if (btrfsic_submit_bio_wait(WRITE_SYNC, bio))
ff023aac
SB		 4281 goto leave_with_eio;
4282
4283 bio_put(bio);
4284 return 0;
4285}
Linux kernel - Deliverables
This page took 0.433281 seconds and 5 git commands to generate.