projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Linux 3.13-rc1
[deliverable/linux.git] / fs / btrfs / scrub.c
CommitLineData
a2de733c 1/*
b6bfebc1 2 * Copyright (C) 2011, 2012 STRATO. All rights reserved.
a2de733c
AJ		 3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
17 */
18
a2de733c 19#include <linux/blkdev.h>
558540c1 20#include <linux/ratelimit.h>
a2de733c
AJ		 21#include "ctree.h"
22#include "volumes.h"
23#include "disk-io.h"
24#include "ordered-data.h"
0ef8e451 25#include "transaction.h"
558540c1 26#include "backref.h"
5da6fcbc 27#include "extent_io.h"
ff023aac 28#include "dev-replace.h"
21adbd5c 29#include "check-integrity.h"
606686ee 30#include "rcu-string.h"
53b381b3 31#include "raid56.h"
a2de733c
AJ		 32
33/*
34 * This is only the first step towards a full-features scrub. It reads all
35 * extent and super block and verifies the checksums. In case a bad checksum
36 * is found or the extent cannot be read, good data will be written back if
37 * any can be found.
38 *
39 * Future enhancements:
a2de733c
AJ		 40 * - In case an unrepairable extent is encountered, track which files are
41 * affected and report them
a2de733c 42 * - track and record media errors, throw out bad devices
a2de733c 43 * - add a mode to also read unallocated space
a2de733c
AJ		 44 */
45
b5d67f64 46struct scrub_block;
d9d181c1 47struct scrub_ctx;
a2de733c 48
ff023aac
SB		 49/*
50 * the following three values only influence the performance.
51 * The last one configures the number of parallel and outstanding I/O
52 * operations. The first two values configure an upper limit for the number
53 * of (dynamically allocated) pages that are added to a bio.
54 */
55#define SCRUB_PAGES_PER_RD_BIO 32 /* 128k per bio */
56#define SCRUB_PAGES_PER_WR_BIO 32 /* 128k per bio */
57#define SCRUB_BIOS_PER_SCTX 64 /* 8MB per device in flight */
7a9e9987
SB		 58
59/*
60 * the following value times PAGE_SIZE needs to be large enough to match the
61 * largest node/leaf/sector size that shall be supported.
62 * Values larger than BTRFS_STRIPE_LEN are not supported.
63 */
b5d67f64 64#define SCRUB_MAX_PAGES_PER_BLOCK 16 /* 64k per node/leaf/sector */
a2de733c
AJ		 65
66struct scrub_page {
b5d67f64
SB		 67 struct scrub_block *sblock;
68 struct page *page;
442a4f63 69 struct btrfs_device *dev;
a2de733c
AJ		 70 u64 flags; /* extent flags */
71 u64 generation;
b5d67f64
SB		 72 u64 logical;
73 u64 physical;
ff023aac 74 u64 physical_for_dev_replace;
7a9e9987 75 atomic_t ref_count;
b5d67f64
SB		 76 struct {
77 unsigned int mirror_num:8;
78 unsigned int have_csum:1;
79 unsigned int io_error:1;
80 };
a2de733c
AJ		 81 u8 csum[BTRFS_CSUM_SIZE];
82};
83
84struct scrub_bio {
85 int index;
d9d181c1 86 struct scrub_ctx *sctx;
a36cf8b8 87 struct btrfs_device *dev;
a2de733c
AJ		 88 struct bio *bio;
89 int err;
90 u64 logical;
91 u64 physical;
ff023aac
SB		 92#if SCRUB_PAGES_PER_WR_BIO >= SCRUB_PAGES_PER_RD_BIO
93 struct scrub_page *pagev[SCRUB_PAGES_PER_WR_BIO];
94#else
95 struct scrub_page *pagev[SCRUB_PAGES_PER_RD_BIO];
96#endif
b5d67f64 97 int page_count;
a2de733c
AJ		 98 int next_free;
99 struct btrfs_work work;
100};
101
b5d67f64 102struct scrub_block {
7a9e9987 103 struct scrub_page *pagev[SCRUB_MAX_PAGES_PER_BLOCK];
b5d67f64
SB		 104 int page_count;
105 atomic_t outstanding_pages;
106 atomic_t ref_count; /* free mem on transition to zero */
d9d181c1 107 struct scrub_ctx *sctx;
b5d67f64
SB		 108 struct {
109 unsigned int header_error:1;
110 unsigned int checksum_error:1;
111 unsigned int no_io_error_seen:1;
442a4f63 112 unsigned int generation_error:1; /* also sets header_error */
b5d67f64
SB		 113 };
114};
115
ff023aac
SB		 116struct scrub_wr_ctx {
117 struct scrub_bio *wr_curr_bio;
118 struct btrfs_device *tgtdev;
119 int pages_per_wr_bio; /* <= SCRUB_PAGES_PER_WR_BIO */
120 atomic_t flush_all_writes;
121 struct mutex wr_lock;
122};
123
d9d181c1 124struct scrub_ctx {
ff023aac 125 struct scrub_bio *bios[SCRUB_BIOS_PER_SCTX];
a36cf8b8 126 struct btrfs_root *dev_root;
a2de733c
AJ		 127 int first_free;
128 int curr;
b6bfebc1
SB		 129 atomic_t bios_in_flight;
130 atomic_t workers_pending;
a2de733c
AJ		 131 spinlock_t list_lock;
132 wait_queue_head_t list_wait;
133 u16 csum_size;
134 struct list_head csum_list;
135 atomic_t cancel_req;
8628764e 136 int readonly;
ff023aac 137 int pages_per_rd_bio;
b5d67f64
SB		 138 u32 sectorsize;
139 u32 nodesize;
140 u32 leafsize;
63a212ab
SB		 141
142 int is_dev_replace;
ff023aac 143 struct scrub_wr_ctx wr_ctx;
63a212ab 144
a2de733c
AJ		 145 /*
146 * statistics
147 */
148 struct btrfs_scrub_progress stat;
149 spinlock_t stat_lock;
150};
151
0ef8e451 152struct scrub_fixup_nodatasum {
d9d181c1 153 struct scrub_ctx *sctx;
a36cf8b8 154 struct btrfs_device *dev;
0ef8e451
JS		 155 u64 logical;
156 struct btrfs_root *root;
157 struct btrfs_work work;
158 int mirror_num;
159};
160
652f25a2
JB		 161struct scrub_nocow_inode {
162 u64 inum;
163 u64 offset;
164 u64 root;
165 struct list_head list;
166};
167
ff023aac
SB		 168struct scrub_copy_nocow_ctx {
169 struct scrub_ctx *sctx;
170 u64 logical;
171 u64 len;
172 int mirror_num;
173 u64 physical_for_dev_replace;
652f25a2 174 struct list_head inodes;
ff023aac
SB		 175 struct btrfs_work work;
176};
177
558540c1
JS		 178struct scrub_warning {
179 struct btrfs_path *path;
180 u64 extent_item_size;
181 char *scratch_buf;
182 char *msg_buf;
183 const char *errstr;
184 sector_t sector;
185 u64 logical;
186 struct btrfs_device *dev;
187 int msg_bufsize;
188 int scratch_bufsize;
189};
190
b5d67f64 191
b6bfebc1
SB		 192static void scrub_pending_bio_inc(struct scrub_ctx *sctx);
193static void scrub_pending_bio_dec(struct scrub_ctx *sctx);
194static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx);
195static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx);
b5d67f64 196static int scrub_handle_errored_block(struct scrub_block *sblock_to_check);
d9d181c1 197static int scrub_setup_recheck_block(struct scrub_ctx *sctx,
3ec706c8 198 struct btrfs_fs_info *fs_info,
ff023aac 199 struct scrub_block *original_sblock,
b5d67f64 200 u64 length, u64 logical,
ff023aac 201 struct scrub_block *sblocks_for_recheck);
34f5c8e9
SB		 202static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
203 struct scrub_block *sblock, int is_metadata,
204 int have_csum, u8 *csum, u64 generation,
205 u16 csum_size);
b5d67f64
SB		 206static void scrub_recheck_block_checksum(struct btrfs_fs_info *fs_info,
207 struct scrub_block *sblock,
208 int is_metadata, int have_csum,
209 const u8 *csum, u64 generation,
210 u16 csum_size);
211static void scrub_complete_bio_end_io(struct bio *bio, int err);
212static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
213 struct scrub_block *sblock_good,
214 int force_write);
215static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
216 struct scrub_block *sblock_good,
217 int page_num, int force_write);
ff023aac
SB		 218static void scrub_write_block_to_dev_replace(struct scrub_block *sblock);
219static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
220 int page_num);
b5d67f64
SB		 221static int scrub_checksum_data(struct scrub_block *sblock);
222static int scrub_checksum_tree_block(struct scrub_block *sblock);
223static int scrub_checksum_super(struct scrub_block *sblock);
224static void scrub_block_get(struct scrub_block *sblock);
225static void scrub_block_put(struct scrub_block *sblock);
7a9e9987
SB		 226static void scrub_page_get(struct scrub_page *spage);
227static void scrub_page_put(struct scrub_page *spage);
ff023aac
SB		 228static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
229 struct scrub_page *spage);
d9d181c1 230static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
a36cf8b8 231 u64 physical, struct btrfs_device *dev, u64 flags,
ff023aac
SB		 232 u64 gen, int mirror_num, u8 *csum, int force,
233 u64 physical_for_dev_replace);
1623edeb 234static void scrub_bio_end_io(struct bio *bio, int err);
b5d67f64
SB		 235static void scrub_bio_end_io_worker(struct btrfs_work *work);
236static void scrub_block_complete(struct scrub_block *sblock);
ff023aac
SB		 237static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
238 u64 extent_logical, u64 extent_len,
239 u64 *extent_physical,
240 struct btrfs_device **extent_dev,
241 int *extent_mirror_num);
242static int scrub_setup_wr_ctx(struct scrub_ctx *sctx,
243 struct scrub_wr_ctx *wr_ctx,
244 struct btrfs_fs_info *fs_info,
245 struct btrfs_device *dev,
246 int is_dev_replace);
247static void scrub_free_wr_ctx(struct scrub_wr_ctx *wr_ctx);
248static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
249 struct scrub_page *spage);
250static void scrub_wr_submit(struct scrub_ctx *sctx);
251static void scrub_wr_bio_end_io(struct bio *bio, int err);
252static void scrub_wr_bio_end_io_worker(struct btrfs_work *work);
253static int write_page_nocow(struct scrub_ctx *sctx,
254 u64 physical_for_dev_replace, struct page *page);
255static int copy_nocow_pages_for_inode(u64 inum, u64 offset, u64 root,
652f25a2 256 struct scrub_copy_nocow_ctx *ctx);
ff023aac
SB		 257static int copy_nocow_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
258 int mirror_num, u64 physical_for_dev_replace);
259static void copy_nocow_pages_worker(struct btrfs_work *work);
1623edeb
SB		 260
261
b6bfebc1
SB		 262static void scrub_pending_bio_inc(struct scrub_ctx *sctx)
263{
264 atomic_inc(&sctx->bios_in_flight);
265}
266
267static void scrub_pending_bio_dec(struct scrub_ctx *sctx)
268{
269 atomic_dec(&sctx->bios_in_flight);
270 wake_up(&sctx->list_wait);
271}
272
273/*
274 * used for workers that require transaction commits (i.e., for the
275 * NOCOW case)
276 */
277static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx)
278{
279 struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
280
281 /*
282 * increment scrubs_running to prevent cancel requests from
283 * completing as long as a worker is running. we must also
284 * increment scrubs_paused to prevent deadlocking on pause
285 * requests used for transactions commits (as the worker uses a
286 * transaction context). it is safe to regard the worker
287 * as paused for all matters practical. effectively, we only
288 * avoid cancellation requests from completing.
289 */
290 mutex_lock(&fs_info->scrub_lock);
291 atomic_inc(&fs_info->scrubs_running);
292 atomic_inc(&fs_info->scrubs_paused);
293 mutex_unlock(&fs_info->scrub_lock);
294 atomic_inc(&sctx->workers_pending);
295}
296
297/* used for workers that require transaction commits */
298static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx)
299{
300 struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
301
302 /*
303 * see scrub_pending_trans_workers_inc() why we're pretending
304 * to be paused in the scrub counters
305 */
306 mutex_lock(&fs_info->scrub_lock);
307 atomic_dec(&fs_info->scrubs_running);
308 atomic_dec(&fs_info->scrubs_paused);
309 mutex_unlock(&fs_info->scrub_lock);
310 atomic_dec(&sctx->workers_pending);
311 wake_up(&fs_info->scrub_pause_wait);
312 wake_up(&sctx->list_wait);
313}
314
d9d181c1 315static void scrub_free_csums(struct scrub_ctx *sctx)
a2de733c 316{
d9d181c1 317 while (!list_empty(&sctx->csum_list)) {
a2de733c 318 struct btrfs_ordered_sum *sum;
d9d181c1 319 sum = list_first_entry(&sctx->csum_list,
a2de733c
AJ		 320 struct btrfs_ordered_sum, list);
321 list_del(&sum->list);
322 kfree(sum);
323 }
324}
325
d9d181c1 326static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx)
a2de733c
AJ		 327{
328 int i;
a2de733c 329
d9d181c1 330 if (!sctx)
a2de733c
AJ		 331 return;
332
ff023aac
SB		 333 scrub_free_wr_ctx(&sctx->wr_ctx);
334
b5d67f64 335 /* this can happen when scrub is cancelled */
d9d181c1
SB		 336 if (sctx->curr != -1) {
337 struct scrub_bio *sbio = sctx->bios[sctx->curr];
b5d67f64
SB		 338
339 for (i = 0; i < sbio->page_count; i++) {
ff023aac 340 WARN_ON(!sbio->pagev[i]->page);
b5d67f64
SB		 341 scrub_block_put(sbio->pagev[i]->sblock);
342 }
343 bio_put(sbio->bio);
344 }
345
ff023aac 346 for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
d9d181c1 347 struct scrub_bio *sbio = sctx->bios[i];
a2de733c
AJ		 348
349 if (!sbio)
350 break;
a2de733c
AJ		 351 kfree(sbio);
352 }
353
d9d181c1
SB		 354 scrub_free_csums(sctx);
355 kfree(sctx);
a2de733c
AJ		 356}
357
358static noinline_for_stack
63a212ab 359struct scrub_ctx *scrub_setup_ctx(struct btrfs_device *dev, int is_dev_replace)
a2de733c 360{
d9d181c1 361 struct scrub_ctx *sctx;
a2de733c 362 int i;
a2de733c 363 struct btrfs_fs_info *fs_info = dev->dev_root->fs_info;
ff023aac
SB		 364 int pages_per_rd_bio;
365 int ret;
a2de733c 366
ff023aac
SB		 367 /*
368 * the setting of pages_per_rd_bio is correct for scrub but might
369 * be wrong for the dev_replace code where we might read from
370 * different devices in the initial huge bios. However, that
371 * code is able to correctly handle the case when adding a page
372 * to a bio fails.
373 */
374 if (dev->bdev)
375 pages_per_rd_bio = min_t(int, SCRUB_PAGES_PER_RD_BIO,
376 bio_get_nr_vecs(dev->bdev));
377 else
378 pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO;
d9d181c1
SB		 379 sctx = kzalloc(sizeof(*sctx), GFP_NOFS);
380 if (!sctx)
a2de733c 381 goto nomem;
63a212ab 382 sctx->is_dev_replace = is_dev_replace;
ff023aac 383 sctx->pages_per_rd_bio = pages_per_rd_bio;
d9d181c1 384 sctx->curr = -1;
a36cf8b8 385 sctx->dev_root = dev->dev_root;
ff023aac 386 for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
a2de733c
AJ		 387 struct scrub_bio *sbio;
388
389 sbio = kzalloc(sizeof(*sbio), GFP_NOFS);
390 if (!sbio)
391 goto nomem;
d9d181c1 392 sctx->bios[i] = sbio;
a2de733c 393
a2de733c 394 sbio->index = i;
d9d181c1 395 sbio->sctx = sctx;
b5d67f64
SB		 396 sbio->page_count = 0;
397 sbio->work.func = scrub_bio_end_io_worker;
a2de733c 398
ff023aac 399 if (i != SCRUB_BIOS_PER_SCTX - 1)
d9d181c1 400 sctx->bios[i]->next_free = i + 1;
0ef8e451 401 else
d9d181c1
SB		 402 sctx->bios[i]->next_free = -1;
403 }
404 sctx->first_free = 0;
405 sctx->nodesize = dev->dev_root->nodesize;
406 sctx->leafsize = dev->dev_root->leafsize;
407 sctx->sectorsize = dev->dev_root->sectorsize;
b6bfebc1
SB		 408 atomic_set(&sctx->bios_in_flight, 0);
409 atomic_set(&sctx->workers_pending, 0);
d9d181c1
SB		 410 atomic_set(&sctx->cancel_req, 0);
411 sctx->csum_size = btrfs_super_csum_size(fs_info->super_copy);
412 INIT_LIST_HEAD(&sctx->csum_list);
413
414 spin_lock_init(&sctx->list_lock);
415 spin_lock_init(&sctx->stat_lock);
416 init_waitqueue_head(&sctx->list_wait);
ff023aac
SB		 417
418 ret = scrub_setup_wr_ctx(sctx, &sctx->wr_ctx, fs_info,
419 fs_info->dev_replace.tgtdev, is_dev_replace);
420 if (ret) {
421 scrub_free_ctx(sctx);
422 return ERR_PTR(ret);
423 }
d9d181c1 424 return sctx;
a2de733c
AJ		 425
426nomem:
d9d181c1 427 scrub_free_ctx(sctx);
a2de733c
AJ		 428 return ERR_PTR(-ENOMEM);
429}
430
ff023aac
SB		 431static int scrub_print_warning_inode(u64 inum, u64 offset, u64 root,
432 void *warn_ctx)
558540c1
JS		 433{
434 u64 isize;
435 u32 nlink;
436 int ret;
437 int i;
438 struct extent_buffer *eb;
439 struct btrfs_inode_item *inode_item;
ff023aac 440 struct scrub_warning *swarn = warn_ctx;
558540c1
JS		 441 struct btrfs_fs_info *fs_info = swarn->dev->dev_root->fs_info;
442 struct inode_fs_paths *ipath = NULL;
443 struct btrfs_root *local_root;
444 struct btrfs_key root_key;
445
446 root_key.objectid = root;
447 root_key.type = BTRFS_ROOT_ITEM_KEY;
448 root_key.offset = (u64)-1;
449 local_root = btrfs_read_fs_root_no_name(fs_info, &root_key);
450 if (IS_ERR(local_root)) {
451 ret = PTR_ERR(local_root);
452 goto err;
453 }
454
455 ret = inode_item_info(inum, 0, local_root, swarn->path);
456 if (ret) {
457 btrfs_release_path(swarn->path);
458 goto err;
459 }
460
461 eb = swarn->path->nodes[0];
462 inode_item = btrfs_item_ptr(eb, swarn->path->slots[0],
463 struct btrfs_inode_item);
464 isize = btrfs_inode_size(eb, inode_item);
465 nlink = btrfs_inode_nlink(eb, inode_item);
466 btrfs_release_path(swarn->path);
467
468 ipath = init_ipath(4096, local_root, swarn->path);
26bdef54
DC		 469 if (IS_ERR(ipath)) {
470 ret = PTR_ERR(ipath);
471 ipath = NULL;
472 goto err;
473 }
558540c1
JS		 474 ret = paths_from_inode(inum, ipath);
475
476 if (ret < 0)
477 goto err;
478
479 /*
480 * we deliberately ignore the bit ipath might have been too small to
481 * hold all of the paths here
482 */
483 for (i = 0; i < ipath->fspath->elem_cnt; ++i)
606686ee 484 printk_in_rcu(KERN_WARNING "btrfs: %s at logical %llu on dev "
558540c1
JS		 485 "%s, sector %llu, root %llu, inode %llu, offset %llu, "
486 "length %llu, links %u (path: %s)\n", swarn->errstr,
606686ee 487 swarn->logical, rcu_str_deref(swarn->dev->name),
558540c1
JS		 488 (unsigned long long)swarn->sector, root, inum, offset,
489 min(isize - offset, (u64)PAGE_SIZE), nlink,
745c4d8e 490 (char *)(unsigned long)ipath->fspath->val[i]);
558540c1
JS		 491
492 free_ipath(ipath);
493 return 0;
494
495err:
606686ee 496 printk_in_rcu(KERN_WARNING "btrfs: %s at logical %llu on dev "
558540c1
JS		 497 "%s, sector %llu, root %llu, inode %llu, offset %llu: path "
498 "resolving failed with ret=%d\n", swarn->errstr,
606686ee 499 swarn->logical, rcu_str_deref(swarn->dev->name),
558540c1
JS		 500 (unsigned long long)swarn->sector, root, inum, offset, ret);
501
502 free_ipath(ipath);
503 return 0;
504}
505
b5d67f64 506static void scrub_print_warning(const char *errstr, struct scrub_block *sblock)
558540c1 507{
a36cf8b8
SB		 508 struct btrfs_device *dev;
509 struct btrfs_fs_info *fs_info;
558540c1
JS		 510 struct btrfs_path *path;
511 struct btrfs_key found_key;
512 struct extent_buffer *eb;
513 struct btrfs_extent_item *ei;
514 struct scrub_warning swarn;
69917e43
LB		 515 unsigned long ptr = 0;
516 u64 extent_item_pos;
517 u64 flags = 0;
558540c1 518 u64 ref_root;
69917e43 519 u32 item_size;
558540c1 520 u8 ref_level;
558540c1 521 const int bufsize = 4096;
69917e43 522 int ret;
558540c1 523
a36cf8b8 524 WARN_ON(sblock->page_count < 1);
7a9e9987 525 dev = sblock->pagev[0]->dev;
a36cf8b8
SB		 526 fs_info = sblock->sctx->dev_root->fs_info;
527
558540c1
JS		 528 path = btrfs_alloc_path();
529
530 swarn.scratch_buf = kmalloc(bufsize, GFP_NOFS);
531 swarn.msg_buf = kmalloc(bufsize, GFP_NOFS);
7a9e9987
SB		 532 swarn.sector = (sblock->pagev[0]->physical) >> 9;
533 swarn.logical = sblock->pagev[0]->logical;
558540c1 534 swarn.errstr = errstr;
a36cf8b8 535 swarn.dev = NULL;
558540c1
JS		 536 swarn.msg_bufsize = bufsize;
537 swarn.scratch_bufsize = bufsize;
538
539 if (!path || !swarn.scratch_buf || !swarn.msg_buf)
540 goto out;
541
69917e43
LB		 542 ret = extent_from_logical(fs_info, swarn.logical, path, &found_key,
543 &flags);
558540c1
JS		 544 if (ret < 0)
545 goto out;
546
4692cf58 547 extent_item_pos = swarn.logical - found_key.objectid;
558540c1
JS		 548 swarn.extent_item_size = found_key.offset;
549
550 eb = path->nodes[0];
551 ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item);
552 item_size = btrfs_item_size_nr(eb, path->slots[0]);
553
69917e43 554 if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
558540c1
JS		 555 do {
556 ret = tree_backref_for_extent(&ptr, eb, ei, item_size,
557 &ref_root, &ref_level);
606686ee 558 printk_in_rcu(KERN_WARNING
1623edeb 559 "btrfs: %s at logical %llu on dev %s, "
558540c1 560 "sector %llu: metadata %s (level %d) in tree "
606686ee
JB		 561 "%llu\n", errstr, swarn.logical,
562 rcu_str_deref(dev->name),
558540c1
JS		 563 (unsigned long long)swarn.sector,
564 ref_level ? "node" : "leaf",
565 ret < 0 ? -1 : ref_level,
566 ret < 0 ? -1 : ref_root);
567 } while (ret != 1);
d8fe29e9 568 btrfs_release_path(path);
558540c1 569 } else {
d8fe29e9 570 btrfs_release_path(path);
558540c1 571 swarn.path = path;
a36cf8b8 572 swarn.dev = dev;
7a3ae2f8
JS		 573 iterate_extent_inodes(fs_info, found_key.objectid,
574 extent_item_pos, 1,
558540c1
JS		 575 scrub_print_warning_inode, &swarn);
576 }
577
578out:
579 btrfs_free_path(path);
580 kfree(swarn.scratch_buf);
581 kfree(swarn.msg_buf);
582}
583
ff023aac 584static int scrub_fixup_readpage(u64 inum, u64 offset, u64 root, void *fixup_ctx)
0ef8e451 585{
5da6fcbc 586 struct page *page = NULL;
0ef8e451 587 unsigned long index;
ff023aac 588 struct scrub_fixup_nodatasum *fixup = fixup_ctx;
0ef8e451 589 int ret;
5da6fcbc 590 int corrected = 0;
0ef8e451 591 struct btrfs_key key;
5da6fcbc 592 struct inode *inode = NULL;
6f1c3605 593 struct btrfs_fs_info *fs_info;
0ef8e451
JS		 594 u64 end = offset + PAGE_SIZE - 1;
595 struct btrfs_root *local_root;
6f1c3605 596 int srcu_index;
0ef8e451
JS		 597
598 key.objectid = root;
599 key.type = BTRFS_ROOT_ITEM_KEY;
600 key.offset = (u64)-1;
6f1c3605
LB		 601
602 fs_info = fixup->root->fs_info;
603 srcu_index = srcu_read_lock(&fs_info->subvol_srcu);
604
605 local_root = btrfs_read_fs_root_no_name(fs_info, &key);
606 if (IS_ERR(local_root)) {
607 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
0ef8e451 608 return PTR_ERR(local_root);
6f1c3605 609 }
0ef8e451
JS		 610
611 key.type = BTRFS_INODE_ITEM_KEY;
612 key.objectid = inum;
613 key.offset = 0;
6f1c3605
LB		 614 inode = btrfs_iget(fs_info->sb, &key, local_root, NULL);
615 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
0ef8e451
JS		 616 if (IS_ERR(inode))
617 return PTR_ERR(inode);
618
0ef8e451
JS		 619 index = offset >> PAGE_CACHE_SHIFT;
620
621 page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
5da6fcbc
JS		 622 if (!page) {
623 ret = -ENOMEM;
624 goto out;
625 }
626
627 if (PageUptodate(page)) {
5da6fcbc
JS		 628 if (PageDirty(page)) {
629 /*
630 * we need to write the data to the defect sector. the
631 * data that was in that sector is not in memory,
632 * because the page was modified. we must not write the
633 * modified page to that sector.
634 *
635 * TODO: what could be done here: wait for the delalloc
636 * runner to write out that page (might involve
637 * COW) and see whether the sector is still
638 * referenced afterwards.
639 *
640 * For the meantime, we'll treat this error
641 * incorrectable, although there is a chance that a
642 * later scrub will find the bad sector again and that
643 * there's no dirty page in memory, then.
644 */
645 ret = -EIO;
646 goto out;
647 }
3ec706c8
SB		 648 fs_info = BTRFS_I(inode)->root->fs_info;
649 ret = repair_io_failure(fs_info, offset, PAGE_SIZE,
5da6fcbc
JS		 650 fixup->logical, page,
651 fixup->mirror_num);
652 unlock_page(page);
653 corrected = !ret;
654 } else {
655 /*
656 * we need to get good data first. the general readpage path
657 * will call repair_io_failure for us, we just have to make
658 * sure we read the bad mirror.
659 */
660 ret = set_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
661 EXTENT_DAMAGED, GFP_NOFS);
662 if (ret) {
663 /* set_extent_bits should give proper error */
664 WARN_ON(ret > 0);
665 if (ret > 0)
666 ret = -EFAULT;
667 goto out;
668 }
669
670 ret = extent_read_full_page(&BTRFS_I(inode)->io_tree, page,
671 btrfs_get_extent,
672 fixup->mirror_num);
673 wait_on_page_locked(page);
674
675 corrected = !test_range_bit(&BTRFS_I(inode)->io_tree, offset,
676 end, EXTENT_DAMAGED, 0, NULL);
677 if (!corrected)
678 clear_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
679 EXTENT_DAMAGED, GFP_NOFS);
680 }
681
682out:
683 if (page)
684 put_page(page);
685 if (inode)
686 iput(inode);
0ef8e451
JS		 687
688 if (ret < 0)
689 return ret;
690
691 if (ret == 0 && corrected) {
692 /*
693 * we only need to call readpage for one of the inodes belonging
694 * to this extent. so make iterate_extent_inodes stop
695 */
696 return 1;
697 }
698
699 return -EIO;
700}
701
702static void scrub_fixup_nodatasum(struct btrfs_work *work)
703{
704 int ret;
705 struct scrub_fixup_nodatasum *fixup;
d9d181c1 706 struct scrub_ctx *sctx;
0ef8e451
JS		 707 struct btrfs_trans_handle *trans = NULL;
708 struct btrfs_fs_info *fs_info;
709 struct btrfs_path *path;
710 int uncorrectable = 0;
711
712 fixup = container_of(work, struct scrub_fixup_nodatasum, work);
d9d181c1 713 sctx = fixup->sctx;
0ef8e451
JS		 714 fs_info = fixup->root->fs_info;
715
716 path = btrfs_alloc_path();
717 if (!path) {
d9d181c1
SB		 718 spin_lock(&sctx->stat_lock);
719 ++sctx->stat.malloc_errors;
720 spin_unlock(&sctx->stat_lock);
0ef8e451
JS		 721 uncorrectable = 1;
722 goto out;
723 }
724
725 trans = btrfs_join_transaction(fixup->root);
726 if (IS_ERR(trans)) {
727 uncorrectable = 1;
728 goto out;
729 }
730
731 /*
732 * the idea is to trigger a regular read through the standard path. we
733 * read a page from the (failed) logical address by specifying the
734 * corresponding copynum of the failed sector. thus, that readpage is
735 * expected to fail.
736 * that is the point where on-the-fly error correction will kick in
737 * (once it's finished) and rewrite the failed sector if a good copy
738 * can be found.
739 */
740 ret = iterate_inodes_from_logical(fixup->logical, fixup->root->fs_info,
741 path, scrub_fixup_readpage,
742 fixup);
743 if (ret < 0) {
744 uncorrectable = 1;
745 goto out;
746 }
747 WARN_ON(ret != 1);
748
d9d181c1
SB		 749 spin_lock(&sctx->stat_lock);
750 ++sctx->stat.corrected_errors;
751 spin_unlock(&sctx->stat_lock);
0ef8e451
JS		 752
753out:
754 if (trans && !IS_ERR(trans))
755 btrfs_end_transaction(trans, fixup->root);
756 if (uncorrectable) {
d9d181c1
SB		 757 spin_lock(&sctx->stat_lock);
758 ++sctx->stat.uncorrectable_errors;
759 spin_unlock(&sctx->stat_lock);
ff023aac
SB		 760 btrfs_dev_replace_stats_inc(
761 &sctx->dev_root->fs_info->dev_replace.
762 num_uncorrectable_read_errors);
606686ee 763 printk_ratelimited_in_rcu(KERN_ERR
b5d67f64 764 "btrfs: unable to fixup (nodatasum) error at logical %llu on dev %s\n",
c1c9ff7c 765 fixup->logical, rcu_str_deref(fixup->dev->name));
0ef8e451
JS		 766 }
767
768 btrfs_free_path(path);
769 kfree(fixup);
770
b6bfebc1 771 scrub_pending_trans_workers_dec(sctx);
0ef8e451
JS		 772}
773
a2de733c 774/*
b5d67f64
SB		 775 * scrub_handle_errored_block gets called when either verification of the
776 * pages failed or the bio failed to read, e.g. with EIO. In the latter
777 * case, this function handles all pages in the bio, even though only one
778 * may be bad.
779 * The goal of this function is to repair the errored block by using the
780 * contents of one of the mirrors.
a2de733c 781 */
b5d67f64 782static int scrub_handle_errored_block(struct scrub_block *sblock_to_check)
a2de733c 783{
d9d181c1 784 struct scrub_ctx *sctx = sblock_to_check->sctx;
a36cf8b8 785 struct btrfs_device *dev;
b5d67f64
SB		 786 struct btrfs_fs_info *fs_info;
787 u64 length;
788 u64 logical;
789 u64 generation;
790 unsigned int failed_mirror_index;
791 unsigned int is_metadata;
792 unsigned int have_csum;
793 u8 *csum;
794 struct scrub_block *sblocks_for_recheck; /* holds one for each mirror */
795 struct scrub_block *sblock_bad;
796 int ret;
797 int mirror_index;
798 int page_num;
799 int success;
558540c1 800 static DEFINE_RATELIMIT_STATE(_rs, DEFAULT_RATELIMIT_INTERVAL,
b5d67f64
SB		 801 DEFAULT_RATELIMIT_BURST);
802
803 BUG_ON(sblock_to_check->page_count < 1);
a36cf8b8 804 fs_info = sctx->dev_root->fs_info;
4ded4f63
SB		 805 if (sblock_to_check->pagev[0]->flags & BTRFS_EXTENT_FLAG_SUPER) {
806 /*
807 * if we find an error in a super block, we just report it.
808 * They will get written with the next transaction commit
809 * anyway
810 */
811 spin_lock(&sctx->stat_lock);
812 ++sctx->stat.super_errors;
813 spin_unlock(&sctx->stat_lock);
814 return 0;
815 }
b5d67f64 816 length = sblock_to_check->page_count * PAGE_SIZE;
7a9e9987
SB		 817 logical = sblock_to_check->pagev[0]->logical;
818 generation = sblock_to_check->pagev[0]->generation;
819 BUG_ON(sblock_to_check->pagev[0]->mirror_num < 1);
820 failed_mirror_index = sblock_to_check->pagev[0]->mirror_num - 1;
821 is_metadata = !(sblock_to_check->pagev[0]->flags &
b5d67f64 822 BTRFS_EXTENT_FLAG_DATA);
7a9e9987
SB		 823 have_csum = sblock_to_check->pagev[0]->have_csum;
824 csum = sblock_to_check->pagev[0]->csum;
825 dev = sblock_to_check->pagev[0]->dev;
13db62b7 826
ff023aac
SB		 827 if (sctx->is_dev_replace && !is_metadata && !have_csum) {
828 sblocks_for_recheck = NULL;
829 goto nodatasum_case;
830 }
831
b5d67f64
SB		 832 /*
833 * read all mirrors one after the other. This includes to
834 * re-read the extent or metadata block that failed (that was
835 * the cause that this fixup code is called) another time,
836 * page by page this time in order to know which pages
837 * caused I/O errors and which ones are good (for all mirrors).
838 * It is the goal to handle the situation when more than one
839 * mirror contains I/O errors, but the errors do not
840 * overlap, i.e. the data can be repaired by selecting the
841 * pages from those mirrors without I/O error on the
842 * particular pages. One example (with blocks >= 2 * PAGE_SIZE)
843 * would be that mirror #1 has an I/O error on the first page,
844 * the second page is good, and mirror #2 has an I/O error on
845 * the second page, but the first page is good.
846 * Then the first page of the first mirror can be repaired by
847 * taking the first page of the second mirror, and the
848 * second page of the second mirror can be repaired by
849 * copying the contents of the 2nd page of the 1st mirror.
850 * One more note: if the pages of one mirror contain I/O
851 * errors, the checksum cannot be verified. In order to get
852 * the best data for repairing, the first attempt is to find
853 * a mirror without I/O errors and with a validated checksum.
854 * Only if this is not possible, the pages are picked from
855 * mirrors with I/O errors without considering the checksum.
856 * If the latter is the case, at the end, the checksum of the
857 * repaired area is verified in order to correctly maintain
858 * the statistics.
859 */
860
861 sblocks_for_recheck = kzalloc(BTRFS_MAX_MIRRORS *
862 sizeof(*sblocks_for_recheck),
863 GFP_NOFS);
864 if (!sblocks_for_recheck) {
d9d181c1
SB		 865 spin_lock(&sctx->stat_lock);
866 sctx->stat.malloc_errors++;
867 sctx->stat.read_errors++;
868 sctx->stat.uncorrectable_errors++;
869 spin_unlock(&sctx->stat_lock);
a36cf8b8 870 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
b5d67f64 871 goto out;
a2de733c
AJ		 872 }
873
b5d67f64 874 /* setup the context, map the logical blocks and alloc the pages */
ff023aac 875 ret = scrub_setup_recheck_block(sctx, fs_info, sblock_to_check, length,
b5d67f64
SB		 876 logical, sblocks_for_recheck);
877 if (ret) {
d9d181c1
SB		 878 spin_lock(&sctx->stat_lock);
879 sctx->stat.read_errors++;
880 sctx->stat.uncorrectable_errors++;
881 spin_unlock(&sctx->stat_lock);
a36cf8b8 882 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
b5d67f64
SB		 883 goto out;
884 }
885 BUG_ON(failed_mirror_index >= BTRFS_MAX_MIRRORS);
886 sblock_bad = sblocks_for_recheck + failed_mirror_index;
13db62b7 887
b5d67f64 888 /* build and submit the bios for the failed mirror, check checksums */
34f5c8e9
SB		 889 scrub_recheck_block(fs_info, sblock_bad, is_metadata, have_csum,
890 csum, generation, sctx->csum_size);
a2de733c 891
b5d67f64
SB		 892 if (!sblock_bad->header_error && !sblock_bad->checksum_error &&
893 sblock_bad->no_io_error_seen) {
894 /*
895 * the error disappeared after reading page by page, or
896 * the area was part of a huge bio and other parts of the
897 * bio caused I/O errors, or the block layer merged several
898 * read requests into one and the error is caused by a
899 * different bio (usually one of the two latter cases is
900 * the cause)
901 */
d9d181c1
SB		 902 spin_lock(&sctx->stat_lock);
903 sctx->stat.unverified_errors++;
904 spin_unlock(&sctx->stat_lock);
a2de733c 905
ff023aac
SB		 906 if (sctx->is_dev_replace)
907 scrub_write_block_to_dev_replace(sblock_bad);
b5d67f64 908 goto out;
a2de733c 909 }
a2de733c 910
b5d67f64 911 if (!sblock_bad->no_io_error_seen) {
d9d181c1
SB		 912 spin_lock(&sctx->stat_lock);
913 sctx->stat.read_errors++;
914 spin_unlock(&sctx->stat_lock);
b5d67f64
SB		 915 if (__ratelimit(&_rs))
916 scrub_print_warning("i/o error", sblock_to_check);
a36cf8b8 917 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
b5d67f64 918 } else if (sblock_bad->checksum_error) {
d9d181c1
SB		 919 spin_lock(&sctx->stat_lock);
920 sctx->stat.csum_errors++;
921 spin_unlock(&sctx->stat_lock);
b5d67f64
SB		 922 if (__ratelimit(&_rs))
923 scrub_print_warning("checksum error", sblock_to_check);
a36cf8b8 924 btrfs_dev_stat_inc_and_print(dev,
442a4f63 925 BTRFS_DEV_STAT_CORRUPTION_ERRS);
b5d67f64 926 } else if (sblock_bad->header_error) {
d9d181c1
SB		 927 spin_lock(&sctx->stat_lock);
928 sctx->stat.verify_errors++;
929 spin_unlock(&sctx->stat_lock);
b5d67f64
SB		 930 if (__ratelimit(&_rs))
931 scrub_print_warning("checksum/header error",
932 sblock_to_check);
442a4f63 933 if (sblock_bad->generation_error)
a36cf8b8 934 btrfs_dev_stat_inc_and_print(dev,
442a4f63
SB		 935 BTRFS_DEV_STAT_GENERATION_ERRS);
936 else
a36cf8b8 937 btrfs_dev_stat_inc_and_print(dev,
442a4f63 938 BTRFS_DEV_STAT_CORRUPTION_ERRS);
b5d67f64 939 }
a2de733c 940
33ef30ad
ID		 941 if (sctx->readonly) {
942 ASSERT(!sctx->is_dev_replace);
943 goto out;
944 }
a2de733c 945
b5d67f64
SB		 946 if (!is_metadata && !have_csum) {
947 struct scrub_fixup_nodatasum *fixup_nodatasum;
a2de733c 948
ff023aac
SB		 949nodatasum_case:
950 WARN_ON(sctx->is_dev_replace);
951
b5d67f64
SB		 952 /*
953 * !is_metadata and !have_csum, this means that the data
954 * might not be COW'ed, that it might be modified
955 * concurrently. The general strategy to work on the
956 * commit root does not help in the case when COW is not
957 * used.
958 */
959 fixup_nodatasum = kzalloc(sizeof(*fixup_nodatasum), GFP_NOFS);
960 if (!fixup_nodatasum)
961 goto did_not_correct_error;
d9d181c1 962 fixup_nodatasum->sctx = sctx;
a36cf8b8 963 fixup_nodatasum->dev = dev;
b5d67f64
SB		 964 fixup_nodatasum->logical = logical;
965 fixup_nodatasum->root = fs_info->extent_root;
966 fixup_nodatasum->mirror_num = failed_mirror_index + 1;
b6bfebc1 967 scrub_pending_trans_workers_inc(sctx);
b5d67f64
SB		 968 fixup_nodatasum->work.func = scrub_fixup_nodatasum;
969 btrfs_queue_worker(&fs_info->scrub_workers,
970 &fixup_nodatasum->work);
971 goto out;
a2de733c
AJ		 972 }
973
b5d67f64
SB		 974 /*
975 * now build and submit the bios for the other mirrors, check
cb2ced73
SB		 976 * checksums.
977 * First try to pick the mirror which is completely without I/O
b5d67f64
SB		 978 * errors and also does not have a checksum error.
979 * If one is found, and if a checksum is present, the full block
980 * that is known to contain an error is rewritten. Afterwards
981 * the block is known to be corrected.
982 * If a mirror is found which is completely correct, and no
983 * checksum is present, only those pages are rewritten that had
984 * an I/O error in the block to be repaired, since it cannot be
985 * determined, which copy of the other pages is better (and it
986 * could happen otherwise that a correct page would be
987 * overwritten by a bad one).
988 */
989 for (mirror_index = 0;
990 mirror_index < BTRFS_MAX_MIRRORS &&
991 sblocks_for_recheck[mirror_index].page_count > 0;
992 mirror_index++) {
cb2ced73 993 struct scrub_block *sblock_other;
b5d67f64 994
cb2ced73
SB		 995 if (mirror_index == failed_mirror_index)
996 continue;
997 sblock_other = sblocks_for_recheck + mirror_index;
998
999 /* build and submit the bios, check checksums */
34f5c8e9
SB		 1000 scrub_recheck_block(fs_info, sblock_other, is_metadata,
1001 have_csum, csum, generation,
1002 sctx->csum_size);
1003
1004 if (!sblock_other->header_error &&
b5d67f64
SB		 1005 !sblock_other->checksum_error &&
1006 sblock_other->no_io_error_seen) {
ff023aac
SB		 1007 if (sctx->is_dev_replace) {
1008 scrub_write_block_to_dev_replace(sblock_other);
1009 } else {
1010 int force_write = is_metadata || have_csum;
1011
1012 ret = scrub_repair_block_from_good_copy(
1013 sblock_bad, sblock_other,
1014 force_write);
1015 }
b5d67f64
SB		 1016 if (0 == ret)
1017 goto corrected_error;
1018 }
1019 }
a2de733c
AJ		 1020
1021 /*
ff023aac
SB		 1022 * for dev_replace, pick good pages and write to the target device.
1023 */
1024 if (sctx->is_dev_replace) {
1025 success = 1;
1026 for (page_num = 0; page_num < sblock_bad->page_count;
1027 page_num++) {
1028 int sub_success;
1029
1030 sub_success = 0;
1031 for (mirror_index = 0;
1032 mirror_index < BTRFS_MAX_MIRRORS &&
1033 sblocks_for_recheck[mirror_index].page_count > 0;
1034 mirror_index++) {
1035 struct scrub_block *sblock_other =
1036 sblocks_for_recheck + mirror_index;
1037 struct scrub_page *page_other =
1038 sblock_other->pagev[page_num];
1039
1040 if (!page_other->io_error) {
1041 ret = scrub_write_page_to_dev_replace(
1042 sblock_other, page_num);
1043 if (ret == 0) {
1044 /* succeeded for this page */
1045 sub_success = 1;
1046 break;
1047 } else {
1048 btrfs_dev_replace_stats_inc(
1049 &sctx->dev_root->
1050 fs_info->dev_replace.
1051 num_write_errors);
1052 }
1053 }
1054 }
1055
1056 if (!sub_success) {
1057 /*
1058 * did not find a mirror to fetch the page
1059 * from. scrub_write_page_to_dev_replace()
1060 * handles this case (page->io_error), by
1061 * filling the block with zeros before
1062 * submitting the write request
1063 */
1064 success = 0;
1065 ret = scrub_write_page_to_dev_replace(
1066 sblock_bad, page_num);
1067 if (ret)
1068 btrfs_dev_replace_stats_inc(
1069 &sctx->dev_root->fs_info->
1070 dev_replace.num_write_errors);
1071 }
1072 }
1073
1074 goto out;
1075 }
1076
1077 /*
1078 * for regular scrub, repair those pages that are errored.
1079 * In case of I/O errors in the area that is supposed to be
b5d67f64
SB		 1080 * repaired, continue by picking good copies of those pages.
1081 * Select the good pages from mirrors to rewrite bad pages from
1082 * the area to fix. Afterwards verify the checksum of the block
1083 * that is supposed to be repaired. This verification step is
1084 * only done for the purpose of statistic counting and for the
1085 * final scrub report, whether errors remain.
1086 * A perfect algorithm could make use of the checksum and try
1087 * all possible combinations of pages from the different mirrors
1088 * until the checksum verification succeeds. For example, when
1089 * the 2nd page of mirror #1 faces I/O errors, and the 2nd page
1090 * of mirror #2 is readable but the final checksum test fails,
1091 * then the 2nd page of mirror #3 could be tried, whether now
1092 * the final checksum succeedes. But this would be a rare
1093 * exception and is therefore not implemented. At least it is
1094 * avoided that the good copy is overwritten.
1095 * A more useful improvement would be to pick the sectors
1096 * without I/O error based on sector sizes (512 bytes on legacy
1097 * disks) instead of on PAGE_SIZE. Then maybe 512 byte of one
1098 * mirror could be repaired by taking 512 byte of a different
1099 * mirror, even if other 512 byte sectors in the same PAGE_SIZE
1100 * area are unreadable.
a2de733c 1101 */
a2de733c 1102
b5d67f64
SB		 1103 /* can only fix I/O errors from here on */
1104 if (sblock_bad->no_io_error_seen)
1105 goto did_not_correct_error;
1106
1107 success = 1;
1108 for (page_num = 0; page_num < sblock_bad->page_count; page_num++) {
7a9e9987 1109 struct scrub_page *page_bad = sblock_bad->pagev[page_num];
b5d67f64
SB		 1110
1111 if (!page_bad->io_error)
a2de733c 1112 continue;
b5d67f64
SB		 1113
1114 for (mirror_index = 0;
1115 mirror_index < BTRFS_MAX_MIRRORS &&
1116 sblocks_for_recheck[mirror_index].page_count > 0;
1117 mirror_index++) {
1118 struct scrub_block *sblock_other = sblocks_for_recheck +
1119 mirror_index;
7a9e9987
SB		 1120 struct scrub_page *page_other = sblock_other->pagev[
1121 page_num];
b5d67f64
SB		 1122
1123 if (!page_other->io_error) {
1124 ret = scrub_repair_page_from_good_copy(
1125 sblock_bad, sblock_other, page_num, 0);
1126 if (0 == ret) {
1127 page_bad->io_error = 0;
1128 break; /* succeeded for this page */
1129 }
1130 }
96e36920 1131 }
a2de733c 1132
b5d67f64
SB		 1133 if (page_bad->io_error) {
1134 /* did not find a mirror to copy the page from */
1135 success = 0;
1136 }
a2de733c 1137 }
a2de733c 1138
b5d67f64
SB		 1139 if (success) {
1140 if (is_metadata || have_csum) {
1141 /*
1142 * need to verify the checksum now that all
1143 * sectors on disk are repaired (the write
1144 * request for data to be repaired is on its way).
1145 * Just be lazy and use scrub_recheck_block()
1146 * which re-reads the data before the checksum
1147 * is verified, but most likely the data comes out
1148 * of the page cache.
1149 */
34f5c8e9
SB		 1150 scrub_recheck_block(fs_info, sblock_bad,
1151 is_metadata, have_csum, csum,
1152 generation, sctx->csum_size);
1153 if (!sblock_bad->header_error &&
b5d67f64
SB		 1154 !sblock_bad->checksum_error &&
1155 sblock_bad->no_io_error_seen)
1156 goto corrected_error;
1157 else
1158 goto did_not_correct_error;
1159 } else {
1160corrected_error:
d9d181c1
SB		 1161 spin_lock(&sctx->stat_lock);
1162 sctx->stat.corrected_errors++;
1163 spin_unlock(&sctx->stat_lock);
606686ee 1164 printk_ratelimited_in_rcu(KERN_ERR
b5d67f64 1165 "btrfs: fixed up error at logical %llu on dev %s\n",
c1c9ff7c 1166 logical, rcu_str_deref(dev->name));
8628764e 1167 }
b5d67f64
SB		 1168 } else {
1169did_not_correct_error:
d9d181c1
SB		 1170 spin_lock(&sctx->stat_lock);
1171 sctx->stat.uncorrectable_errors++;
1172 spin_unlock(&sctx->stat_lock);
606686ee 1173 printk_ratelimited_in_rcu(KERN_ERR
b5d67f64 1174 "btrfs: unable to fixup (regular) error at logical %llu on dev %s\n",
c1c9ff7c 1175 logical, rcu_str_deref(dev->name));
96e36920 1176 }
a2de733c 1177
b5d67f64
SB		 1178out:
1179 if (sblocks_for_recheck) {
1180 for (mirror_index = 0; mirror_index < BTRFS_MAX_MIRRORS;
1181 mirror_index++) {
1182 struct scrub_block *sblock = sblocks_for_recheck +
1183 mirror_index;
1184 int page_index;
1185
7a9e9987
SB		 1186 for (page_index = 0; page_index < sblock->page_count;
1187 page_index++) {
1188 sblock->pagev[page_index]->sblock = NULL;
1189 scrub_page_put(sblock->pagev[page_index]);
1190 }
b5d67f64
SB		 1191 }
1192 kfree(sblocks_for_recheck);
1193 }
a2de733c 1194
b5d67f64
SB		 1195 return 0;
1196}
a2de733c 1197
d9d181c1 1198static int scrub_setup_recheck_block(struct scrub_ctx *sctx,
3ec706c8 1199 struct btrfs_fs_info *fs_info,
ff023aac 1200 struct scrub_block *original_sblock,
b5d67f64
SB		 1201 u64 length, u64 logical,
1202 struct scrub_block *sblocks_for_recheck)
1203{
1204 int page_index;
1205 int mirror_index;
1206 int ret;
1207
1208 /*
7a9e9987 1209 * note: the two members ref_count and outstanding_pages
b5d67f64
SB		 1210 * are not used (and not set) in the blocks that are used for
1211 * the recheck procedure
1212 */
1213
1214 page_index = 0;
1215 while (length > 0) {
1216 u64 sublen = min_t(u64, length, PAGE_SIZE);
1217 u64 mapped_length = sublen;
1218 struct btrfs_bio *bbio = NULL;
a2de733c 1219
b5d67f64
SB		 1220 /*
1221 * with a length of PAGE_SIZE, each returned stripe
1222 * represents one mirror
1223 */
29a8d9a0
SB		 1224 ret = btrfs_map_block(fs_info, REQ_GET_READ_MIRRORS, logical,
1225 &mapped_length, &bbio, 0);
b5d67f64
SB		 1226 if (ret || !bbio || mapped_length < sublen) {
1227 kfree(bbio);
1228 return -EIO;
1229 }
a2de733c 1230
ff023aac 1231 BUG_ON(page_index >= SCRUB_PAGES_PER_RD_BIO);
b5d67f64
SB		 1232 for (mirror_index = 0; mirror_index < (int)bbio->num_stripes;
1233 mirror_index++) {
1234 struct scrub_block *sblock;
1235 struct scrub_page *page;
1236
1237 if (mirror_index >= BTRFS_MAX_MIRRORS)
1238 continue;
1239
1240 sblock = sblocks_for_recheck + mirror_index;
7a9e9987
SB		 1241 sblock->sctx = sctx;
1242 page = kzalloc(sizeof(*page), GFP_NOFS);
1243 if (!page) {
1244leave_nomem:
d9d181c1
SB		 1245 spin_lock(&sctx->stat_lock);
1246 sctx->stat.malloc_errors++;
1247 spin_unlock(&sctx->stat_lock);
cf93dcce 1248 kfree(bbio);
b5d67f64
SB		 1249 return -ENOMEM;
1250 }
7a9e9987
SB		 1251 scrub_page_get(page);
1252 sblock->pagev[page_index] = page;
1253 page->logical = logical;
1254 page->physical = bbio->stripes[mirror_index].physical;
ff023aac
SB		 1255 BUG_ON(page_index >= original_sblock->page_count);
1256 page->physical_for_dev_replace =
1257 original_sblock->pagev[page_index]->
1258 physical_for_dev_replace;
7a9e9987
SB		 1259 /* for missing devices, dev->bdev is NULL */
1260 page->dev = bbio->stripes[mirror_index].dev;
1261 page->mirror_num = mirror_index + 1;
b5d67f64 1262 sblock->page_count++;
7a9e9987
SB		 1263 page->page = alloc_page(GFP_NOFS);
1264 if (!page->page)
1265 goto leave_nomem;
b5d67f64
SB		 1266 }
1267 kfree(bbio);
1268 length -= sublen;
1269 logical += sublen;
1270 page_index++;
1271 }
1272
1273 return 0;
96e36920
ID		 1274}
1275
b5d67f64
SB		 1276/*
1277 * this function will check the on disk data for checksum errors, header
1278 * errors and read I/O errors. If any I/O errors happen, the exact pages
1279 * which are errored are marked as being bad. The goal is to enable scrub
1280 * to take those pages that are not errored from all the mirrors so that
1281 * the pages that are errored in the just handled mirror can be repaired.
1282 */
34f5c8e9
SB		 1283static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
1284 struct scrub_block *sblock, int is_metadata,
1285 int have_csum, u8 *csum, u64 generation,
1286 u16 csum_size)
96e36920 1287{
b5d67f64 1288 int page_num;
96e36920 1289
b5d67f64
SB		 1290 sblock->no_io_error_seen = 1;
1291 sblock->header_error = 0;
1292 sblock->checksum_error = 0;
96e36920 1293
b5d67f64
SB		 1294 for (page_num = 0; page_num < sblock->page_count; page_num++) {
1295 struct bio *bio;
7a9e9987 1296 struct scrub_page *page = sblock->pagev[page_num];
b5d67f64
SB		 1297 DECLARE_COMPLETION_ONSTACK(complete);
1298
442a4f63 1299 if (page->dev->bdev == NULL) {
ea9947b4
SB		 1300 page->io_error = 1;
1301 sblock->no_io_error_seen = 0;
1302 continue;
1303 }
1304
7a9e9987 1305 WARN_ON(!page->page);
9be3395b 1306 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
34f5c8e9
SB		 1307 if (!bio) {
1308 page->io_error = 1;
1309 sblock->no_io_error_seen = 0;
1310 continue;
1311 }
442a4f63 1312 bio->bi_bdev = page->dev->bdev;
b5d67f64
SB		 1313 bio->bi_sector = page->physical >> 9;
1314 bio->bi_end_io = scrub_complete_bio_end_io;
1315 bio->bi_private = &complete;
1316
34f5c8e9 1317 bio_add_page(bio, page->page, PAGE_SIZE, 0);
b5d67f64 1318 btrfsic_submit_bio(READ, bio);
96e36920 1319
b5d67f64
SB		 1320 /* this will also unplug the queue */
1321 wait_for_completion(&complete);
96e36920 1322
b5d67f64
SB		 1323 page->io_error = !test_bit(BIO_UPTODATE, &bio->bi_flags);
1324 if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
1325 sblock->no_io_error_seen = 0;
1326 bio_put(bio);
1327 }
96e36920 1328
b5d67f64
SB		 1329 if (sblock->no_io_error_seen)
1330 scrub_recheck_block_checksum(fs_info, sblock, is_metadata,
1331 have_csum, csum, generation,
1332 csum_size);
1333
34f5c8e9 1334 return;
a2de733c
AJ		 1335}
1336
b5d67f64
SB		 1337static void scrub_recheck_block_checksum(struct btrfs_fs_info *fs_info,
1338 struct scrub_block *sblock,
1339 int is_metadata, int have_csum,
1340 const u8 *csum, u64 generation,
1341 u16 csum_size)
a2de733c 1342{
b5d67f64
SB		 1343 int page_num;
1344 u8 calculated_csum[BTRFS_CSUM_SIZE];
1345 u32 crc = ~(u32)0;
b5d67f64
SB		 1346 void *mapped_buffer;
1347
7a9e9987 1348 WARN_ON(!sblock->pagev[0]->page);
b5d67f64
SB		 1349 if (is_metadata) {
1350 struct btrfs_header *h;
1351
7a9e9987 1352 mapped_buffer = kmap_atomic(sblock->pagev[0]->page);
b5d67f64
SB		 1353 h = (struct btrfs_header *)mapped_buffer;
1354
3cae210f 1355 if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h) ||
b5d67f64
SB		 1356 memcmp(h->fsid, fs_info->fsid, BTRFS_UUID_SIZE) ||
1357 memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
442a4f63 1358 BTRFS_UUID_SIZE)) {
b5d67f64 1359 sblock->header_error = 1;
3cae210f 1360 } else if (generation != btrfs_stack_header_generation(h)) {
442a4f63
SB		 1361 sblock->header_error = 1;
1362 sblock->generation_error = 1;
1363 }
b5d67f64
SB		 1364 csum = h->csum;
1365 } else {
1366 if (!have_csum)
1367 return;
a2de733c 1368
7a9e9987 1369 mapped_buffer = kmap_atomic(sblock->pagev[0]->page);
b5d67f64 1370 }
a2de733c 1371
b5d67f64
SB		 1372 for (page_num = 0;;) {
1373 if (page_num == 0 && is_metadata)
b0496686 1374 crc = btrfs_csum_data(
b5d67f64
SB		 1375 ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE,
1376 crc, PAGE_SIZE - BTRFS_CSUM_SIZE);
1377 else
b0496686 1378 crc = btrfs_csum_data(mapped_buffer, crc, PAGE_SIZE);
b5d67f64 1379
9613bebb 1380 kunmap_atomic(mapped_buffer);
b5d67f64
SB		 1381 page_num++;
1382 if (page_num >= sblock->page_count)
1383 break;
7a9e9987 1384 WARN_ON(!sblock->pagev[page_num]->page);
b5d67f64 1385
7a9e9987 1386 mapped_buffer = kmap_atomic(sblock->pagev[page_num]->page);
b5d67f64
SB		 1387 }
1388
1389 btrfs_csum_final(crc, calculated_csum);
1390 if (memcmp(calculated_csum, csum, csum_size))
1391 sblock->checksum_error = 1;
a2de733c
AJ		 1392}
1393
b5d67f64 1394static void scrub_complete_bio_end_io(struct bio *bio, int err)
a2de733c 1395{
b5d67f64
SB		 1396 complete((struct completion *)bio->bi_private);
1397}
a2de733c 1398
b5d67f64
SB		 1399static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
1400 struct scrub_block *sblock_good,
1401 int force_write)
1402{
1403 int page_num;
1404 int ret = 0;
96e36920 1405
b5d67f64
SB		 1406 for (page_num = 0; page_num < sblock_bad->page_count; page_num++) {
1407 int ret_sub;
96e36920 1408
b5d67f64
SB		 1409 ret_sub = scrub_repair_page_from_good_copy(sblock_bad,
1410 sblock_good,
1411 page_num,
1412 force_write);
1413 if (ret_sub)
1414 ret = ret_sub;
a2de733c 1415 }
b5d67f64
SB		 1416
1417 return ret;
1418}
1419
1420static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
1421 struct scrub_block *sblock_good,
1422 int page_num, int force_write)
1423{
7a9e9987
SB		 1424 struct scrub_page *page_bad = sblock_bad->pagev[page_num];
1425 struct scrub_page *page_good = sblock_good->pagev[page_num];
b5d67f64 1426
7a9e9987
SB		 1427 BUG_ON(page_bad->page == NULL);
1428 BUG_ON(page_good->page == NULL);
b5d67f64
SB		 1429 if (force_write || sblock_bad->header_error ||
1430 sblock_bad->checksum_error || page_bad->io_error) {
1431 struct bio *bio;
1432 int ret;
1433 DECLARE_COMPLETION_ONSTACK(complete);
1434
ff023aac
SB		 1435 if (!page_bad->dev->bdev) {
1436 printk_ratelimited(KERN_WARNING
1437 "btrfs: scrub_repair_page_from_good_copy(bdev == NULL) is unexpected!\n");
1438 return -EIO;
1439 }
1440
9be3395b 1441 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
e627ee7b
TI		 1442 if (!bio)
1443 return -EIO;
442a4f63 1444 bio->bi_bdev = page_bad->dev->bdev;
b5d67f64
SB		 1445 bio->bi_sector = page_bad->physical >> 9;
1446 bio->bi_end_io = scrub_complete_bio_end_io;
1447 bio->bi_private = &complete;
1448
1449 ret = bio_add_page(bio, page_good->page, PAGE_SIZE, 0);
1450 if (PAGE_SIZE != ret) {
1451 bio_put(bio);
1452 return -EIO;
13db62b7 1453 }
b5d67f64
SB		 1454 btrfsic_submit_bio(WRITE, bio);
1455
1456 /* this will also unplug the queue */
1457 wait_for_completion(&complete);
442a4f63
SB		 1458 if (!bio_flagged(bio, BIO_UPTODATE)) {
1459 btrfs_dev_stat_inc_and_print(page_bad->dev,
1460 BTRFS_DEV_STAT_WRITE_ERRS);
ff023aac
SB		 1461 btrfs_dev_replace_stats_inc(
1462 &sblock_bad->sctx->dev_root->fs_info->
1463 dev_replace.num_write_errors);
442a4f63
SB		 1464 bio_put(bio);
1465 return -EIO;
1466 }
b5d67f64 1467 bio_put(bio);
a2de733c
AJ		 1468 }
1469
b5d67f64
SB		 1470 return 0;
1471}
1472
ff023aac
SB		 1473static void scrub_write_block_to_dev_replace(struct scrub_block *sblock)
1474{
1475 int page_num;
1476
1477 for (page_num = 0; page_num < sblock->page_count; page_num++) {
1478 int ret;
1479
1480 ret = scrub_write_page_to_dev_replace(sblock, page_num);
1481 if (ret)
1482 btrfs_dev_replace_stats_inc(
1483 &sblock->sctx->dev_root->fs_info->dev_replace.
1484 num_write_errors);
1485 }
1486}
1487
1488static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
1489 int page_num)
1490{
1491 struct scrub_page *spage = sblock->pagev[page_num];
1492
1493 BUG_ON(spage->page == NULL);
1494 if (spage->io_error) {
1495 void *mapped_buffer = kmap_atomic(spage->page);
1496
1497 memset(mapped_buffer, 0, PAGE_CACHE_SIZE);
1498 flush_dcache_page(spage->page);
1499 kunmap_atomic(mapped_buffer);
1500 }
1501 return scrub_add_page_to_wr_bio(sblock->sctx, spage);
1502}
1503
1504static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
1505 struct scrub_page *spage)
1506{
1507 struct scrub_wr_ctx *wr_ctx = &sctx->wr_ctx;
1508 struct scrub_bio *sbio;
1509 int ret;
1510
1511 mutex_lock(&wr_ctx->wr_lock);
1512again:
1513 if (!wr_ctx->wr_curr_bio) {
1514 wr_ctx->wr_curr_bio = kzalloc(sizeof(*wr_ctx->wr_curr_bio),
1515 GFP_NOFS);
1516 if (!wr_ctx->wr_curr_bio) {
1517 mutex_unlock(&wr_ctx->wr_lock);
1518 return -ENOMEM;
1519 }
1520 wr_ctx->wr_curr_bio->sctx = sctx;
1521 wr_ctx->wr_curr_bio->page_count = 0;
1522 }
1523 sbio = wr_ctx->wr_curr_bio;
1524 if (sbio->page_count == 0) {
1525 struct bio *bio;
1526
1527 sbio->physical = spage->physical_for_dev_replace;
1528 sbio->logical = spage->logical;
1529 sbio->dev = wr_ctx->tgtdev;
1530 bio = sbio->bio;
1531 if (!bio) {
9be3395b 1532 bio = btrfs_io_bio_alloc(GFP_NOFS, wr_ctx->pages_per_wr_bio);
ff023aac
SB		 1533 if (!bio) {
1534 mutex_unlock(&wr_ctx->wr_lock);
1535 return -ENOMEM;
1536 }
1537 sbio->bio = bio;
1538 }
1539
1540 bio->bi_private = sbio;
1541 bio->bi_end_io = scrub_wr_bio_end_io;
1542 bio->bi_bdev = sbio->dev->bdev;
1543 bio->bi_sector = sbio->physical >> 9;
1544 sbio->err = 0;
1545 } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
1546 spage->physical_for_dev_replace ||
1547 sbio->logical + sbio->page_count * PAGE_SIZE !=
1548 spage->logical) {
1549 scrub_wr_submit(sctx);
1550 goto again;
1551 }
1552
1553 ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
1554 if (ret != PAGE_SIZE) {
1555 if (sbio->page_count < 1) {
1556 bio_put(sbio->bio);
1557 sbio->bio = NULL;
1558 mutex_unlock(&wr_ctx->wr_lock);
1559 return -EIO;
1560 }
1561 scrub_wr_submit(sctx);
1562 goto again;
1563 }
1564
1565 sbio->pagev[sbio->page_count] = spage;
1566 scrub_page_get(spage);
1567 sbio->page_count++;
1568 if (sbio->page_count == wr_ctx->pages_per_wr_bio)
1569 scrub_wr_submit(sctx);
1570 mutex_unlock(&wr_ctx->wr_lock);
1571
1572 return 0;
1573}
1574
1575static void scrub_wr_submit(struct scrub_ctx *sctx)
1576{
1577 struct scrub_wr_ctx *wr_ctx = &sctx->wr_ctx;
1578 struct scrub_bio *sbio;
1579
1580 if (!wr_ctx->wr_curr_bio)
1581 return;
1582
1583 sbio = wr_ctx->wr_curr_bio;
1584 wr_ctx->wr_curr_bio = NULL;
1585 WARN_ON(!sbio->bio->bi_bdev);
1586 scrub_pending_bio_inc(sctx);
1587 /* process all writes in a single worker thread. Then the block layer
1588 * orders the requests before sending them to the driver which
1589 * doubled the write performance on spinning disks when measured
1590 * with Linux 3.5 */
1591 btrfsic_submit_bio(WRITE, sbio->bio);
1592}
1593
1594static void scrub_wr_bio_end_io(struct bio *bio, int err)
1595{
1596 struct scrub_bio *sbio = bio->bi_private;
1597 struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;
1598
1599 sbio->err = err;
1600 sbio->bio = bio;
1601
1602 sbio->work.func = scrub_wr_bio_end_io_worker;
1603 btrfs_queue_worker(&fs_info->scrub_wr_completion_workers, &sbio->work);
1604}
1605
1606static void scrub_wr_bio_end_io_worker(struct btrfs_work *work)
1607{
1608 struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
1609 struct scrub_ctx *sctx = sbio->sctx;
1610 int i;
1611
1612 WARN_ON(sbio->page_count > SCRUB_PAGES_PER_WR_BIO);
1613 if (sbio->err) {
1614 struct btrfs_dev_replace *dev_replace =
1615 &sbio->sctx->dev_root->fs_info->dev_replace;
1616
1617 for (i = 0; i < sbio->page_count; i++) {
1618 struct scrub_page *spage = sbio->pagev[i];
1619
1620 spage->io_error = 1;
1621 btrfs_dev_replace_stats_inc(&dev_replace->
1622 num_write_errors);
1623 }
1624 }
1625
1626 for (i = 0; i < sbio->page_count; i++)
1627 scrub_page_put(sbio->pagev[i]);
1628
1629 bio_put(sbio->bio);
1630 kfree(sbio);
1631 scrub_pending_bio_dec(sctx);
1632}
1633
1634static int scrub_checksum(struct scrub_block *sblock)
b5d67f64
SB		 1635{
1636 u64 flags;
1637 int ret;
1638
7a9e9987
SB		 1639 WARN_ON(sblock->page_count < 1);
1640 flags = sblock->pagev[0]->flags;
b5d67f64
SB		 1641 ret = 0;
1642 if (flags & BTRFS_EXTENT_FLAG_DATA)
1643 ret = scrub_checksum_data(sblock);
1644 else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)
1645 ret = scrub_checksum_tree_block(sblock);
1646 else if (flags & BTRFS_EXTENT_FLAG_SUPER)
1647 (void)scrub_checksum_super(sblock);
1648 else
1649 WARN_ON(1);
1650 if (ret)
1651 scrub_handle_errored_block(sblock);
ff023aac
SB		 1652
1653 return ret;
a2de733c
AJ		 1654}
1655
b5d67f64 1656static int scrub_checksum_data(struct scrub_block *sblock)
a2de733c 1657{
d9d181c1 1658 struct scrub_ctx *sctx = sblock->sctx;
a2de733c 1659 u8 csum[BTRFS_CSUM_SIZE];
b5d67f64
SB		 1660 u8 *on_disk_csum;
1661 struct page *page;
1662 void *buffer;
a2de733c
AJ		 1663 u32 crc = ~(u32)0;
1664 int fail = 0;
b5d67f64
SB		 1665 u64 len;
1666 int index;
a2de733c 1667
b5d67f64 1668 BUG_ON(sblock->page_count < 1);
7a9e9987 1669 if (!sblock->pagev[0]->have_csum)
a2de733c
AJ		 1670 return 0;
1671
7a9e9987
SB		 1672 on_disk_csum = sblock->pagev[0]->csum;
1673 page = sblock->pagev[0]->page;
9613bebb 1674 buffer = kmap_atomic(page);
b5d67f64 1675
d9d181c1 1676 len = sctx->sectorsize;
b5d67f64
SB		 1677 index = 0;
1678 for (;;) {
1679 u64 l = min_t(u64, len, PAGE_SIZE);
1680
b0496686 1681 crc = btrfs_csum_data(buffer, crc, l);
9613bebb 1682 kunmap_atomic(buffer);
b5d67f64
SB		 1683 len -= l;
1684 if (len == 0)
1685 break;
1686 index++;
1687 BUG_ON(index >= sblock->page_count);
7a9e9987
SB		 1688 BUG_ON(!sblock->pagev[index]->page);
1689 page = sblock->pagev[index]->page;
9613bebb 1690 buffer = kmap_atomic(page);
b5d67f64
SB		 1691 }
1692
a2de733c 1693 btrfs_csum_final(crc, csum);
d9d181c1 1694 if (memcmp(csum, on_disk_csum, sctx->csum_size))
a2de733c
AJ		 1695 fail = 1;
1696
a2de733c
AJ		 1697 return fail;
1698}
1699
b5d67f64 1700static int scrub_checksum_tree_block(struct scrub_block *sblock)
a2de733c 1701{
d9d181c1 1702 struct scrub_ctx *sctx = sblock->sctx;
a2de733c 1703 struct btrfs_header *h;
a36cf8b8 1704 struct btrfs_root *root = sctx->dev_root;
a2de733c 1705 struct btrfs_fs_info *fs_info = root->fs_info;
b5d67f64
SB		 1706 u8 calculated_csum[BTRFS_CSUM_SIZE];
1707 u8 on_disk_csum[BTRFS_CSUM_SIZE];
1708 struct page *page;
1709 void *mapped_buffer;
1710 u64 mapped_size;
1711 void *p;
a2de733c
AJ		 1712 u32 crc = ~(u32)0;
1713 int fail = 0;
1714 int crc_fail = 0;
b5d67f64
SB		 1715 u64 len;
1716 int index;
1717
1718 BUG_ON(sblock->page_count < 1);
7a9e9987 1719 page = sblock->pagev[0]->page;
9613bebb 1720 mapped_buffer = kmap_atomic(page);
b5d67f64 1721 h = (struct btrfs_header *)mapped_buffer;
d9d181c1 1722 memcpy(on_disk_csum, h->csum, sctx->csum_size);
a2de733c
AJ		 1723
1724 /*
1725 * we don't use the getter functions here, as we
1726 * a) don't have an extent buffer and
1727 * b) the page is already kmapped
1728 */
a2de733c 1729
3cae210f 1730 if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h))
a2de733c
AJ		 1731 ++fail;
1732
3cae210f 1733 if (sblock->pagev[0]->generation != btrfs_stack_header_generation(h))
a2de733c
AJ		 1734 ++fail;
1735
1736 if (memcmp(h->fsid, fs_info->fsid, BTRFS_UUID_SIZE))
1737 ++fail;
1738
1739 if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
1740 BTRFS_UUID_SIZE))
1741 ++fail;
1742
ff023aac 1743 WARN_ON(sctx->nodesize != sctx->leafsize);
d9d181c1 1744 len = sctx->nodesize - BTRFS_CSUM_SIZE;
b5d67f64
SB		 1745 mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
1746 p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
1747 index = 0;
1748 for (;;) {
1749 u64 l = min_t(u64, len, mapped_size);
1750
b0496686 1751 crc = btrfs_csum_data(p, crc, l);
9613bebb 1752 kunmap_atomic(mapped_buffer);
b5d67f64
SB		 1753 len -= l;
1754 if (len == 0)
1755 break;
1756 index++;
1757 BUG_ON(index >= sblock->page_count);
7a9e9987
SB		 1758 BUG_ON(!sblock->pagev[index]->page);
1759 page = sblock->pagev[index]->page;
9613bebb 1760 mapped_buffer = kmap_atomic(page);
b5d67f64
SB		 1761 mapped_size = PAGE_SIZE;
1762 p = mapped_buffer;
1763 }
1764
1765 btrfs_csum_final(crc, calculated_csum);
d9d181c1 1766 if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
a2de733c
AJ		 1767 ++crc_fail;
1768
a2de733c
AJ		 1769 return fail || crc_fail;
1770}
1771
b5d67f64 1772static int scrub_checksum_super(struct scrub_block *sblock)
a2de733c
AJ		 1773{
1774 struct btrfs_super_block *s;
d9d181c1 1775 struct scrub_ctx *sctx = sblock->sctx;
a36cf8b8 1776 struct btrfs_root *root = sctx->dev_root;
a2de733c 1777 struct btrfs_fs_info *fs_info = root->fs_info;
b5d67f64
SB		 1778 u8 calculated_csum[BTRFS_CSUM_SIZE];
1779 u8 on_disk_csum[BTRFS_CSUM_SIZE];
1780 struct page *page;
1781 void *mapped_buffer;
1782 u64 mapped_size;
1783 void *p;
a2de733c 1784 u32 crc = ~(u32)0;
442a4f63
SB		 1785 int fail_gen = 0;
1786 int fail_cor = 0;
b5d67f64
SB		 1787 u64 len;
1788 int index;
a2de733c 1789
b5d67f64 1790 BUG_ON(sblock->page_count < 1);
7a9e9987 1791 page = sblock->pagev[0]->page;
9613bebb 1792 mapped_buffer = kmap_atomic(page);
b5d67f64 1793 s = (struct btrfs_super_block *)mapped_buffer;
d9d181c1 1794 memcpy(on_disk_csum, s->csum, sctx->csum_size);
a2de733c 1795
3cae210f 1796 if (sblock->pagev[0]->logical != btrfs_super_bytenr(s))
442a4f63 1797 ++fail_cor;
a2de733c 1798
3cae210f 1799 if (sblock->pagev[0]->generation != btrfs_super_generation(s))
442a4f63 1800 ++fail_gen;
a2de733c
AJ		 1801
1802 if (memcmp(s->fsid, fs_info->fsid, BTRFS_UUID_SIZE))
442a4f63 1803 ++fail_cor;
a2de733c 1804
b5d67f64
SB		 1805 len = BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE;
1806 mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
1807 p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
1808 index = 0;
1809 for (;;) {
1810 u64 l = min_t(u64, len, mapped_size);
1811
b0496686 1812 crc = btrfs_csum_data(p, crc, l);
9613bebb 1813 kunmap_atomic(mapped_buffer);
b5d67f64
SB		 1814 len -= l;
1815 if (len == 0)
1816 break;
1817 index++;
1818 BUG_ON(index >= sblock->page_count);
7a9e9987
SB		 1819 BUG_ON(!sblock->pagev[index]->page);
1820 page = sblock->pagev[index]->page;
9613bebb 1821 mapped_buffer = kmap_atomic(page);
b5d67f64
SB		 1822 mapped_size = PAGE_SIZE;
1823 p = mapped_buffer;
1824 }
1825
1826 btrfs_csum_final(crc, calculated_csum);
d9d181c1 1827 if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
442a4f63 1828 ++fail_cor;
a2de733c 1829
442a4f63 1830 if (fail_cor + fail_gen) {
a2de733c
AJ		 1831 /*
1832 * if we find an error in a super block, we just report it.
1833 * They will get written with the next transaction commit
1834 * anyway
1835 */
d9d181c1
SB		 1836 spin_lock(&sctx->stat_lock);
1837 ++sctx->stat.super_errors;
1838 spin_unlock(&sctx->stat_lock);
442a4f63 1839 if (fail_cor)
7a9e9987 1840 btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
442a4f63
SB		 1841 BTRFS_DEV_STAT_CORRUPTION_ERRS);
1842 else
7a9e9987 1843 btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
442a4f63 1844 BTRFS_DEV_STAT_GENERATION_ERRS);
a2de733c
AJ		 1845 }
1846
442a4f63 1847 return fail_cor + fail_gen;
a2de733c
AJ		 1848}
1849
b5d67f64
SB		 1850static void scrub_block_get(struct scrub_block *sblock)
1851{
1852 atomic_inc(&sblock->ref_count);
1853}
1854
1855static void scrub_block_put(struct scrub_block *sblock)
1856{
1857 if (atomic_dec_and_test(&sblock->ref_count)) {
1858 int i;
1859
1860 for (i = 0; i < sblock->page_count; i++)
7a9e9987 1861 scrub_page_put(sblock->pagev[i]);
b5d67f64
SB		 1862 kfree(sblock);
1863 }
1864}
1865
7a9e9987
SB		 1866static void scrub_page_get(struct scrub_page *spage)
1867{
1868 atomic_inc(&spage->ref_count);
1869}
1870
1871static void scrub_page_put(struct scrub_page *spage)
1872{
1873 if (atomic_dec_and_test(&spage->ref_count)) {
1874 if (spage->page)
1875 __free_page(spage->page);
1876 kfree(spage);
1877 }
1878}
1879
d9d181c1 1880static void scrub_submit(struct scrub_ctx *sctx)
a2de733c
AJ		 1881{
1882 struct scrub_bio *sbio;
1883
d9d181c1 1884 if (sctx->curr == -1)
1623edeb 1885 return;
a2de733c 1886
d9d181c1
SB		 1887 sbio = sctx->bios[sctx->curr];
1888 sctx->curr = -1;
b6bfebc1 1889 scrub_pending_bio_inc(sctx);
a2de733c 1890
ff023aac
SB		 1891 if (!sbio->bio->bi_bdev) {
1892 /*
1893 * this case should not happen. If btrfs_map_block() is
1894 * wrong, it could happen for dev-replace operations on
1895 * missing devices when no mirrors are available, but in
1896 * this case it should already fail the mount.
1897 * This case is handled correctly (but _very_ slowly).
1898 */
1899 printk_ratelimited(KERN_WARNING
1900 "btrfs: scrub_submit(bio bdev == NULL) is unexpected!\n");
1901 bio_endio(sbio->bio, -EIO);
1902 } else {
1903 btrfsic_submit_bio(READ, sbio->bio);
1904 }
a2de733c
AJ		 1905}
1906
ff023aac
SB		 1907static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
1908 struct scrub_page *spage)
a2de733c 1909{
b5d67f64 1910 struct scrub_block *sblock = spage->sblock;
a2de733c 1911 struct scrub_bio *sbio;
69f4cb52 1912 int ret;
a2de733c
AJ		 1913
1914again:
1915 /*
1916 * grab a fresh bio or wait for one to become available
1917 */
d9d181c1
SB		 1918 while (sctx->curr == -1) {
1919 spin_lock(&sctx->list_lock);
1920 sctx->curr = sctx->first_free;
1921 if (sctx->curr != -1) {
1922 sctx->first_free = sctx->bios[sctx->curr]->next_free;
1923 sctx->bios[sctx->curr]->next_free = -1;
1924 sctx->bios[sctx->curr]->page_count = 0;
1925 spin_unlock(&sctx->list_lock);
a2de733c 1926 } else {
d9d181c1
SB		 1927 spin_unlock(&sctx->list_lock);
1928 wait_event(sctx->list_wait, sctx->first_free != -1);
a2de733c
AJ		 1929 }
1930 }
d9d181c1 1931 sbio = sctx->bios[sctx->curr];
b5d67f64 1932 if (sbio->page_count == 0) {
69f4cb52
AJ		 1933 struct bio *bio;
1934
b5d67f64
SB		 1935 sbio->physical = spage->physical;
1936 sbio->logical = spage->logical;
a36cf8b8 1937 sbio->dev = spage->dev;
b5d67f64
SB		 1938 bio = sbio->bio;
1939 if (!bio) {
9be3395b 1940 bio = btrfs_io_bio_alloc(GFP_NOFS, sctx->pages_per_rd_bio);
b5d67f64
SB		 1941 if (!bio)
1942 return -ENOMEM;
1943 sbio->bio = bio;
1944 }
69f4cb52
AJ		 1945
1946 bio->bi_private = sbio;
1947 bio->bi_end_io = scrub_bio_end_io;
a36cf8b8
SB		 1948 bio->bi_bdev = sbio->dev->bdev;
1949 bio->bi_sector = sbio->physical >> 9;
69f4cb52 1950 sbio->err = 0;
b5d67f64
SB		 1951 } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
1952 spage->physical ||
1953 sbio->logical + sbio->page_count * PAGE_SIZE !=
a36cf8b8
SB		 1954 spage->logical ||
1955 sbio->dev != spage->dev) {
d9d181c1 1956 scrub_submit(sctx);
a2de733c
AJ		 1957 goto again;
1958 }
69f4cb52 1959
b5d67f64
SB		 1960 sbio->pagev[sbio->page_count] = spage;
1961 ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
1962 if (ret != PAGE_SIZE) {
1963 if (sbio->page_count < 1) {
1964 bio_put(sbio->bio);
1965 sbio->bio = NULL;
1966 return -EIO;
1967 }
d9d181c1 1968 scrub_submit(sctx);
69f4cb52
AJ		 1969 goto again;
1970 }
1971
ff023aac 1972 scrub_block_get(sblock); /* one for the page added to the bio */
b5d67f64
SB		 1973 atomic_inc(&sblock->outstanding_pages);
1974 sbio->page_count++;
ff023aac 1975 if (sbio->page_count == sctx->pages_per_rd_bio)
d9d181c1 1976 scrub_submit(sctx);
b5d67f64
SB		 1977
1978 return 0;
1979}
1980
d9d181c1 1981static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
a36cf8b8 1982 u64 physical, struct btrfs_device *dev, u64 flags,
ff023aac
SB		 1983 u64 gen, int mirror_num, u8 *csum, int force,
1984 u64 physical_for_dev_replace)
b5d67f64
SB		 1985{
1986 struct scrub_block *sblock;
1987 int index;
1988
1989 sblock = kzalloc(sizeof(*sblock), GFP_NOFS);
1990 if (!sblock) {
d9d181c1
SB		 1991 spin_lock(&sctx->stat_lock);
1992 sctx->stat.malloc_errors++;
1993 spin_unlock(&sctx->stat_lock);
b5d67f64 1994 return -ENOMEM;
a2de733c 1995 }
b5d67f64 1996
7a9e9987
SB		 1997 /* one ref inside this function, plus one for each page added to
1998 * a bio later on */
b5d67f64 1999 atomic_set(&sblock->ref_count, 1);
d9d181c1 2000 sblock->sctx = sctx;
b5d67f64
SB		 2001 sblock->no_io_error_seen = 1;
2002
2003 for (index = 0; len > 0; index++) {
7a9e9987 2004 struct scrub_page *spage;
b5d67f64
SB		 2005 u64 l = min_t(u64, len, PAGE_SIZE);
2006
7a9e9987
SB		 2007 spage = kzalloc(sizeof(*spage), GFP_NOFS);
2008 if (!spage) {
2009leave_nomem:
d9d181c1
SB		 2010 spin_lock(&sctx->stat_lock);
2011 sctx->stat.malloc_errors++;
2012 spin_unlock(&sctx->stat_lock);
7a9e9987 2013 scrub_block_put(sblock);
b5d67f64
SB		 2014 return -ENOMEM;
2015 }
7a9e9987
SB		 2016 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
2017 scrub_page_get(spage);
2018 sblock->pagev[index] = spage;
b5d67f64 2019 spage->sblock = sblock;
a36cf8b8 2020 spage->dev = dev;
b5d67f64
SB		 2021 spage->flags = flags;
2022 spage->generation = gen;
2023 spage->logical = logical;
2024 spage->physical = physical;
ff023aac 2025 spage->physical_for_dev_replace = physical_for_dev_replace;
b5d67f64
SB		 2026 spage->mirror_num = mirror_num;
2027 if (csum) {
2028 spage->have_csum = 1;
d9d181c1 2029 memcpy(spage->csum, csum, sctx->csum_size);
b5d67f64
SB		 2030 } else {
2031 spage->have_csum = 0;
2032 }
2033 sblock->page_count++;
7a9e9987
SB		 2034 spage->page = alloc_page(GFP_NOFS);
2035 if (!spage->page)
2036 goto leave_nomem;
b5d67f64
SB		 2037 len -= l;
2038 logical += l;
2039 physical += l;
ff023aac 2040 physical_for_dev_replace += l;
b5d67f64
SB		 2041 }
2042
7a9e9987 2043 WARN_ON(sblock->page_count == 0);
b5d67f64 2044 for (index = 0; index < sblock->page_count; index++) {
7a9e9987 2045 struct scrub_page *spage = sblock->pagev[index];
1bc87793
AJ		 2046 int ret;
2047
ff023aac 2048 ret = scrub_add_page_to_rd_bio(sctx, spage);
b5d67f64
SB		 2049 if (ret) {
2050 scrub_block_put(sblock);
1bc87793 2051 return ret;
b5d67f64 2052 }
1bc87793 2053 }
a2de733c 2054
b5d67f64 2055 if (force)
d9d181c1 2056 scrub_submit(sctx);
a2de733c 2057
b5d67f64
SB		 2058 /* last one frees, either here or in bio completion for last page */
2059 scrub_block_put(sblock);
a2de733c
AJ		 2060 return 0;
2061}
2062
b5d67f64
SB		 2063static void scrub_bio_end_io(struct bio *bio, int err)
2064{
2065 struct scrub_bio *sbio = bio->bi_private;
a36cf8b8 2066 struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;
b5d67f64
SB		 2067
2068 sbio->err = err;
2069 sbio->bio = bio;
2070
2071 btrfs_queue_worker(&fs_info->scrub_workers, &sbio->work);
2072}
2073
2074static void scrub_bio_end_io_worker(struct btrfs_work *work)
2075{
2076 struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
d9d181c1 2077 struct scrub_ctx *sctx = sbio->sctx;
b5d67f64
SB		 2078 int i;
2079
ff023aac 2080 BUG_ON(sbio->page_count > SCRUB_PAGES_PER_RD_BIO);
b5d67f64
SB		 2081 if (sbio->err) {
2082 for (i = 0; i < sbio->page_count; i++) {
2083 struct scrub_page *spage = sbio->pagev[i];
2084
2085 spage->io_error = 1;
2086 spage->sblock->no_io_error_seen = 0;
2087 }
2088 }
2089
2090 /* now complete the scrub_block items that have all pages completed */
2091 for (i = 0; i < sbio->page_count; i++) {
2092 struct scrub_page *spage = sbio->pagev[i];
2093 struct scrub_block *sblock = spage->sblock;
2094
2095 if (atomic_dec_and_test(&sblock->outstanding_pages))
2096 scrub_block_complete(sblock);
2097 scrub_block_put(sblock);
2098 }
2099
b5d67f64
SB		 2100 bio_put(sbio->bio);
2101 sbio->bio = NULL;
d9d181c1
SB		 2102 spin_lock(&sctx->list_lock);
2103 sbio->next_free = sctx->first_free;
2104 sctx->first_free = sbio->index;
2105 spin_unlock(&sctx->list_lock);
ff023aac
SB		 2106
2107 if (sctx->is_dev_replace &&
2108 atomic_read(&sctx->wr_ctx.flush_all_writes)) {
2109 mutex_lock(&sctx->wr_ctx.wr_lock);
2110 scrub_wr_submit(sctx);
2111 mutex_unlock(&sctx->wr_ctx.wr_lock);
2112 }
2113
b6bfebc1 2114 scrub_pending_bio_dec(sctx);
b5d67f64
SB		 2115}
2116
2117static void scrub_block_complete(struct scrub_block *sblock)
2118{
ff023aac 2119 if (!sblock->no_io_error_seen) {
b5d67f64 2120 scrub_handle_errored_block(sblock);
ff023aac
SB		 2121 } else {
2122 /*
2123 * if has checksum error, write via repair mechanism in
2124 * dev replace case, otherwise write here in dev replace
2125 * case.
2126 */
2127 if (!scrub_checksum(sblock) && sblock->sctx->is_dev_replace)
2128 scrub_write_block_to_dev_replace(sblock);
2129 }
b5d67f64
SB		 2130}
2131
d9d181c1 2132static int scrub_find_csum(struct scrub_ctx *sctx, u64 logical, u64 len,
a2de733c
AJ		 2133 u8 *csum)
2134{
2135 struct btrfs_ordered_sum *sum = NULL;
f51a4a18 2136 unsigned long index;
a2de733c 2137 unsigned long num_sectors;
a2de733c 2138
d9d181c1
SB		 2139 while (!list_empty(&sctx->csum_list)) {
2140 sum = list_first_entry(&sctx->csum_list,
a2de733c
AJ		 2141 struct btrfs_ordered_sum, list);
2142 if (sum->bytenr > logical)
2143 return 0;
2144 if (sum->bytenr + sum->len > logical)
2145 break;
2146
d9d181c1 2147 ++sctx->stat.csum_discards;
a2de733c
AJ		 2148 list_del(&sum->list);
2149 kfree(sum);
2150 sum = NULL;
2151 }
2152 if (!sum)
2153 return 0;
2154
f51a4a18 2155 index = ((u32)(logical - sum->bytenr)) / sctx->sectorsize;
d9d181c1 2156 num_sectors = sum->len / sctx->sectorsize;
f51a4a18
MX		 2157 memcpy(csum, sum->sums + index, sctx->csum_size);
2158 if (index == num_sectors - 1) {
a2de733c
AJ		 2159 list_del(&sum->list);
2160 kfree(sum);
2161 }
f51a4a18 2162 return 1;
a2de733c
AJ		 2163}
2164
2165/* scrub extent tries to collect up to 64 kB for each bio */
d9d181c1 2166static int scrub_extent(struct scrub_ctx *sctx, u64 logical, u64 len,
a36cf8b8 2167 u64 physical, struct btrfs_device *dev, u64 flags,
ff023aac 2168 u64 gen, int mirror_num, u64 physical_for_dev_replace)
a2de733c
AJ		 2169{
2170 int ret;
2171 u8 csum[BTRFS_CSUM_SIZE];
b5d67f64
SB		 2172 u32 blocksize;
2173
2174 if (flags & BTRFS_EXTENT_FLAG_DATA) {
d9d181c1
SB		 2175 blocksize = sctx->sectorsize;
2176 spin_lock(&sctx->stat_lock);
2177 sctx->stat.data_extents_scrubbed++;
2178 sctx->stat.data_bytes_scrubbed += len;
2179 spin_unlock(&sctx->stat_lock);
b5d67f64 2180 } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
ff023aac 2181 WARN_ON(sctx->nodesize != sctx->leafsize);
d9d181c1
SB		 2182 blocksize = sctx->nodesize;
2183 spin_lock(&sctx->stat_lock);
2184 sctx->stat.tree_extents_scrubbed++;
2185 sctx->stat.tree_bytes_scrubbed += len;
2186 spin_unlock(&sctx->stat_lock);
b5d67f64 2187 } else {
d9d181c1 2188 blocksize = sctx->sectorsize;
ff023aac 2189 WARN_ON(1);
b5d67f64 2190 }
a2de733c
AJ		 2191
2192 while (len) {
b5d67f64 2193 u64 l = min_t(u64, len, blocksize);
a2de733c
AJ		 2194 int have_csum = 0;
2195
2196 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2197 /* push csums to sbio */
d9d181c1 2198 have_csum = scrub_find_csum(sctx, logical, l, csum);
a2de733c 2199 if (have_csum == 0)
d9d181c1 2200 ++sctx->stat.no_csum;
ff023aac
SB		 2201 if (sctx->is_dev_replace && !have_csum) {
2202 ret = copy_nocow_pages(sctx, logical, l,
2203 mirror_num,
2204 physical_for_dev_replace);
2205 goto behind_scrub_pages;
2206 }
a2de733c 2207 }
a36cf8b8 2208 ret = scrub_pages(sctx, logical, l, physical, dev, flags, gen,
ff023aac
SB		 2209 mirror_num, have_csum ? csum : NULL, 0,
2210 physical_for_dev_replace);
2211behind_scrub_pages:
a2de733c
AJ		 2212 if (ret)
2213 return ret;
2214 len -= l;
2215 logical += l;
2216 physical += l;
ff023aac 2217 physical_for_dev_replace += l;
a2de733c
AJ		 2218 }
2219 return 0;
2220}
2221
d9d181c1 2222static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx,
a36cf8b8
SB		 2223 struct map_lookup *map,
2224 struct btrfs_device *scrub_dev,
ff023aac
SB		 2225 int num, u64 base, u64 length,
2226 int is_dev_replace)
a2de733c
AJ		 2227{
2228 struct btrfs_path *path;
a36cf8b8 2229 struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
a2de733c
AJ		 2230 struct btrfs_root *root = fs_info->extent_root;
2231 struct btrfs_root *csum_root = fs_info->csum_root;
2232 struct btrfs_extent_item *extent;
e7786c3a 2233 struct blk_plug plug;
a2de733c
AJ		 2234 u64 flags;
2235 int ret;
2236 int slot;
a2de733c 2237 u64 nstripes;
a2de733c
AJ		 2238 struct extent_buffer *l;
2239 struct btrfs_key key;
2240 u64 physical;
2241 u64 logical;
625f1c8d 2242 u64 logic_end;
a2de733c 2243 u64 generation;
e12fa9cd 2244 int mirror_num;
7a26285e
AJ		 2245 struct reada_control *reada1;
2246 struct reada_control *reada2;
2247 struct btrfs_key key_start;
2248 struct btrfs_key key_end;
a2de733c
AJ		 2249 u64 increment = map->stripe_len;
2250 u64 offset;
ff023aac
SB		 2251 u64 extent_logical;
2252 u64 extent_physical;
2253 u64 extent_len;
2254 struct btrfs_device *extent_dev;
2255 int extent_mirror_num;
625f1c8d 2256 int stop_loop;
a2de733c 2257
53b381b3
DW		 2258 if (map->type & (BTRFS_BLOCK_GROUP_RAID5 |
2259 BTRFS_BLOCK_GROUP_RAID6)) {
2260 if (num >= nr_data_stripes(map)) {
2261 return 0;
2262 }
2263 }
2264
a2de733c
AJ		 2265 nstripes = length;
2266 offset = 0;
2267 do_div(nstripes, map->stripe_len);
2268 if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
2269 offset = map->stripe_len * num;
2270 increment = map->stripe_len * map->num_stripes;
193ea74b 2271 mirror_num = 1;
a2de733c
AJ		 2272 } else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
2273 int factor = map->num_stripes / map->sub_stripes;
2274 offset = map->stripe_len * (num / map->sub_stripes);
2275 increment = map->stripe_len * factor;
193ea74b 2276 mirror_num = num % map->sub_stripes + 1;
a2de733c
AJ		 2277 } else if (map->type & BTRFS_BLOCK_GROUP_RAID1) {
2278 increment = map->stripe_len;
193ea74b 2279 mirror_num = num % map->num_stripes + 1;
a2de733c
AJ		 2280 } else if (map->type & BTRFS_BLOCK_GROUP_DUP) {
2281 increment = map->stripe_len;
193ea74b 2282 mirror_num = num % map->num_stripes + 1;
a2de733c
AJ		 2283 } else {
2284 increment = map->stripe_len;
193ea74b 2285 mirror_num = 1;
a2de733c
AJ		 2286 }
2287
2288 path = btrfs_alloc_path();
2289 if (!path)
2290 return -ENOMEM;
2291
b5d67f64
SB		 2292 /*
2293 * work on commit root. The related disk blocks are static as
2294 * long as COW is applied. This means, it is save to rewrite
2295 * them to repair disk errors without any race conditions
2296 */
a2de733c
AJ		 2297 path->search_commit_root = 1;
2298 path->skip_locking = 1;
2299
2300 /*
7a26285e
AJ		 2301 * trigger the readahead for extent tree csum tree and wait for
2302 * completion. During readahead, the scrub is officially paused
2303 * to not hold off transaction commits
a2de733c
AJ		 2304 */
2305 logical = base + offset;
a2de733c 2306
d9d181c1 2307 wait_event(sctx->list_wait,
b6bfebc1 2308 atomic_read(&sctx->bios_in_flight) == 0);
7a26285e
AJ		 2309 atomic_inc(&fs_info->scrubs_paused);
2310 wake_up(&fs_info->scrub_pause_wait);
2311
2312 /* FIXME it might be better to start readahead at commit root */
2313 key_start.objectid = logical;
2314 key_start.type = BTRFS_EXTENT_ITEM_KEY;
2315 key_start.offset = (u64)0;
2316 key_end.objectid = base + offset + nstripes * increment;
3173a18f
JB		 2317 key_end.type = BTRFS_METADATA_ITEM_KEY;
2318 key_end.offset = (u64)-1;
7a26285e
AJ		 2319 reada1 = btrfs_reada_add(root, &key_start, &key_end);
2320
2321 key_start.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
2322 key_start.type = BTRFS_EXTENT_CSUM_KEY;
2323 key_start.offset = logical;
2324 key_end.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
2325 key_end.type = BTRFS_EXTENT_CSUM_KEY;
2326 key_end.offset = base + offset + nstripes * increment;
2327 reada2 = btrfs_reada_add(csum_root, &key_start, &key_end);
2328
2329 if (!IS_ERR(reada1))
2330 btrfs_reada_wait(reada1);
2331 if (!IS_ERR(reada2))
2332 btrfs_reada_wait(reada2);
2333
2334 mutex_lock(&fs_info->scrub_lock);
2335 while (atomic_read(&fs_info->scrub_pause_req)) {
2336 mutex_unlock(&fs_info->scrub_lock);
2337 wait_event(fs_info->scrub_pause_wait,
2338 atomic_read(&fs_info->scrub_pause_req) == 0);
2339 mutex_lock(&fs_info->scrub_lock);
a2de733c 2340 }
7a26285e
AJ		 2341 atomic_dec(&fs_info->scrubs_paused);
2342 mutex_unlock(&fs_info->scrub_lock);
2343 wake_up(&fs_info->scrub_pause_wait);
a2de733c
AJ		 2344
2345 /*
2346 * collect all data csums for the stripe to avoid seeking during
2347 * the scrub. This might currently (crc32) end up to be about 1MB
2348 */
e7786c3a 2349 blk_start_plug(&plug);
a2de733c 2350
a2de733c
AJ		 2351 /*
2352 * now find all extents for each stripe and scrub them
2353 */
7a26285e
AJ		 2354 logical = base + offset;
2355 physical = map->stripes[num].physical;
625f1c8d 2356 logic_end = logical + increment * nstripes;
a2de733c 2357 ret = 0;
625f1c8d 2358 while (logical < logic_end) {
a2de733c
AJ		 2359 /*
2360 * canceled?
2361 */
2362 if (atomic_read(&fs_info->scrub_cancel_req) ||
d9d181c1 2363 atomic_read(&sctx->cancel_req)) {
a2de733c
AJ		 2364 ret = -ECANCELED;
2365 goto out;
2366 }
2367 /*
2368 * check to see if we have to pause
2369 */
2370 if (atomic_read(&fs_info->scrub_pause_req)) {
2371 /* push queued extents */
ff023aac 2372 atomic_set(&sctx->wr_ctx.flush_all_writes, 1);
d9d181c1 2373 scrub_submit(sctx);
ff023aac
SB		 2374 mutex_lock(&sctx->wr_ctx.wr_lock);
2375 scrub_wr_submit(sctx);
2376 mutex_unlock(&sctx->wr_ctx.wr_lock);
d9d181c1 2377 wait_event(sctx->list_wait,
b6bfebc1 2378 atomic_read(&sctx->bios_in_flight) == 0);
ff023aac 2379 atomic_set(&sctx->wr_ctx.flush_all_writes, 0);
a2de733c
AJ		 2380 atomic_inc(&fs_info->scrubs_paused);
2381 wake_up(&fs_info->scrub_pause_wait);
2382 mutex_lock(&fs_info->scrub_lock);
2383 while (atomic_read(&fs_info->scrub_pause_req)) {
2384 mutex_unlock(&fs_info->scrub_lock);
2385 wait_event(fs_info->scrub_pause_wait,
2386 atomic_read(&fs_info->scrub_pause_req) == 0);
2387 mutex_lock(&fs_info->scrub_lock);
2388 }
2389 atomic_dec(&fs_info->scrubs_paused);
2390 mutex_unlock(&fs_info->scrub_lock);
2391 wake_up(&fs_info->scrub_pause_wait);
a2de733c
AJ		 2392 }
2393
2394 key.objectid = logical;
2395 key.type = BTRFS_EXTENT_ITEM_KEY;
625f1c8d 2396 key.offset = (u64)-1;
a2de733c
AJ		 2397
2398 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2399 if (ret < 0)
2400 goto out;
3173a18f 2401
8c51032f 2402 if (ret > 0) {
a2de733c
AJ		 2403 ret = btrfs_previous_item(root, path, 0,
2404 BTRFS_EXTENT_ITEM_KEY);
2405 if (ret < 0)
2406 goto out;
8c51032f
AJ		 2407 if (ret > 0) {
2408 /* there's no smaller item, so stick with the
2409 * larger one */
2410 btrfs_release_path(path);
2411 ret = btrfs_search_slot(NULL, root, &key,
2412 path, 0, 0);
2413 if (ret < 0)
2414 goto out;
2415 }
a2de733c
AJ		 2416 }
2417
625f1c8d 2418 stop_loop = 0;
a2de733c 2419 while (1) {
3173a18f
JB		 2420 u64 bytes;
2421
a2de733c
AJ		 2422 l = path->nodes[0];
2423 slot = path->slots[0];
2424 if (slot >= btrfs_header_nritems(l)) {
2425 ret = btrfs_next_leaf(root, path);
2426 if (ret == 0)
2427 continue;
2428 if (ret < 0)
2429 goto out;
2430
625f1c8d 2431 stop_loop = 1;
a2de733c
AJ		 2432 break;
2433 }
2434 btrfs_item_key_to_cpu(l, &key, slot);
2435
3173a18f
JB		 2436 if (key.type == BTRFS_METADATA_ITEM_KEY)
2437 bytes = root->leafsize;
2438 else
2439 bytes = key.offset;
2440
2441 if (key.objectid + bytes <= logical)
a2de733c
AJ		 2442 goto next;
2443
625f1c8d
LB		 2444 if (key.type != BTRFS_EXTENT_ITEM_KEY &&
2445 key.type != BTRFS_METADATA_ITEM_KEY)
2446 goto next;
a2de733c 2447
625f1c8d
LB		 2448 if (key.objectid >= logical + map->stripe_len) {
2449 /* out of this device extent */
2450 if (key.objectid >= logic_end)
2451 stop_loop = 1;
2452 break;
2453 }
a2de733c
AJ		 2454
2455 extent = btrfs_item_ptr(l, slot,
2456 struct btrfs_extent_item);
2457 flags = btrfs_extent_flags(l, extent);
2458 generation = btrfs_extent_generation(l, extent);
2459
2460 if (key.objectid < logical &&
2461 (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)) {
2462 printk(KERN_ERR
2463 "btrfs scrub: tree block %llu spanning "
2464 "stripes, ignored. logical=%llu\n",
c1c9ff7c 2465 key.objectid, logical);
a2de733c
AJ		 2466 goto next;
2467 }
2468
625f1c8d
LB		 2469again:
2470 extent_logical = key.objectid;
2471 extent_len = bytes;
2472
a2de733c
AJ		 2473 /*
2474 * trim extent to this stripe
2475 */
625f1c8d
LB		 2476 if (extent_logical < logical) {
2477 extent_len -= logical - extent_logical;
2478 extent_logical = logical;
a2de733c 2479 }
625f1c8d 2480 if (extent_logical + extent_len >
a2de733c 2481 logical + map->stripe_len) {
625f1c8d
LB		 2482 extent_len = logical + map->stripe_len -
2483 extent_logical;
a2de733c
AJ		 2484 }
2485
625f1c8d 2486 extent_physical = extent_logical - logical + physical;
ff023aac
SB		 2487 extent_dev = scrub_dev;
2488 extent_mirror_num = mirror_num;
2489 if (is_dev_replace)
2490 scrub_remap_extent(fs_info, extent_logical,
2491 extent_len, &extent_physical,
2492 &extent_dev,
2493 &extent_mirror_num);
625f1c8d
LB		 2494
2495 ret = btrfs_lookup_csums_range(csum_root, logical,
2496 logical + map->stripe_len - 1,
2497 &sctx->csum_list, 1);
2498 if (ret)
2499 goto out;
2500
ff023aac
SB		 2501 ret = scrub_extent(sctx, extent_logical, extent_len,
2502 extent_physical, extent_dev, flags,
2503 generation, extent_mirror_num,
115930cb 2504 extent_logical - logical + physical);
a2de733c
AJ		 2505 if (ret)
2506 goto out;
2507
d88d46c6 2508 scrub_free_csums(sctx);
625f1c8d
LB		 2509 if (extent_logical + extent_len <
2510 key.objectid + bytes) {
2511 logical += increment;
2512 physical += map->stripe_len;
2513
2514 if (logical < key.objectid + bytes) {
2515 cond_resched();
2516 goto again;
2517 }
2518
2519 if (logical >= logic_end) {
2520 stop_loop = 1;
2521 break;
2522 }
2523 }
a2de733c
AJ		 2524next:
2525 path->slots[0]++;
2526 }
71267333 2527 btrfs_release_path(path);
a2de733c
AJ		 2528 logical += increment;
2529 physical += map->stripe_len;
d9d181c1 2530 spin_lock(&sctx->stat_lock);
625f1c8d
LB		 2531 if (stop_loop)
2532 sctx->stat.last_physical = map->stripes[num].physical +
2533 length;
2534 else
2535 sctx->stat.last_physical = physical;
d9d181c1 2536 spin_unlock(&sctx->stat_lock);
625f1c8d
LB		 2537 if (stop_loop)
2538 break;
a2de733c 2539 }
ff023aac 2540out:
a2de733c 2541 /* push queued extents */
d9d181c1 2542 scrub_submit(sctx);
ff023aac
SB		 2543 mutex_lock(&sctx->wr_ctx.wr_lock);
2544 scrub_wr_submit(sctx);
2545 mutex_unlock(&sctx->wr_ctx.wr_lock);
a2de733c 2546
e7786c3a 2547 blk_finish_plug(&plug);
a2de733c
AJ		 2548 btrfs_free_path(path);
2549 return ret < 0 ? ret : 0;
2550}
2551
d9d181c1 2552static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx,
a36cf8b8
SB		 2553 struct btrfs_device *scrub_dev,
2554 u64 chunk_tree, u64 chunk_objectid,
2555 u64 chunk_offset, u64 length,
ff023aac 2556 u64 dev_offset, int is_dev_replace)
a2de733c
AJ		 2557{
2558 struct btrfs_mapping_tree *map_tree =
a36cf8b8 2559 &sctx->dev_root->fs_info->mapping_tree;
a2de733c
AJ		 2560 struct map_lookup *map;
2561 struct extent_map *em;
2562 int i;
ff023aac 2563 int ret = 0;
a2de733c
AJ		 2564
2565 read_lock(&map_tree->map_tree.lock);
2566 em = lookup_extent_mapping(&map_tree->map_tree, chunk_offset, 1);
2567 read_unlock(&map_tree->map_tree.lock);
2568
2569 if (!em)
2570 return -EINVAL;
2571
2572 map = (struct map_lookup *)em->bdev;
2573 if (em->start != chunk_offset)
2574 goto out;
2575
2576 if (em->len < length)
2577 goto out;
2578
2579 for (i = 0; i < map->num_stripes; ++i) {
a36cf8b8 2580 if (map->stripes[i].dev->bdev == scrub_dev->bdev &&
859acaf1 2581 map->stripes[i].physical == dev_offset) {
a36cf8b8 2582 ret = scrub_stripe(sctx, map, scrub_dev, i,
ff023aac
SB		 2583 chunk_offset, length,
2584 is_dev_replace);
a2de733c
AJ		 2585 if (ret)
2586 goto out;
2587 }
2588 }
2589out:
2590 free_extent_map(em);
2591
2592 return ret;
2593}
2594
2595static noinline_for_stack
a36cf8b8 2596int scrub_enumerate_chunks(struct scrub_ctx *sctx,
ff023aac
SB		 2597 struct btrfs_device *scrub_dev, u64 start, u64 end,
2598 int is_dev_replace)
a2de733c
AJ		 2599{
2600 struct btrfs_dev_extent *dev_extent = NULL;
2601 struct btrfs_path *path;
a36cf8b8 2602 struct btrfs_root *root = sctx->dev_root;
a2de733c
AJ		 2603 struct btrfs_fs_info *fs_info = root->fs_info;
2604 u64 length;
2605 u64 chunk_tree;
2606 u64 chunk_objectid;
2607 u64 chunk_offset;
2608 int ret;
2609 int slot;
2610 struct extent_buffer *l;
2611 struct btrfs_key key;
2612 struct btrfs_key found_key;
2613 struct btrfs_block_group_cache *cache;
ff023aac 2614 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
a2de733c
AJ		 2615
2616 path = btrfs_alloc_path();
2617 if (!path)
2618 return -ENOMEM;
2619
2620 path->reada = 2;
2621 path->search_commit_root = 1;
2622 path->skip_locking = 1;
2623
a36cf8b8 2624 key.objectid = scrub_dev->devid;
a2de733c
AJ		 2625 key.offset = 0ull;
2626 key.type = BTRFS_DEV_EXTENT_KEY;
2627
a2de733c
AJ		 2628 while (1) {
2629 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2630 if (ret < 0)
8c51032f
AJ		 2631 break;
2632 if (ret > 0) {
2633 if (path->slots[0] >=
2634 btrfs_header_nritems(path->nodes[0])) {
2635 ret = btrfs_next_leaf(root, path);
2636 if (ret)
2637 break;
2638 }
2639 }
a2de733c
AJ		 2640
2641 l = path->nodes[0];
2642 slot = path->slots[0];
2643
2644 btrfs_item_key_to_cpu(l, &found_key, slot);
2645
a36cf8b8 2646 if (found_key.objectid != scrub_dev->devid)
a2de733c
AJ		 2647 break;
2648
8c51032f 2649 if (btrfs_key_type(&found_key) != BTRFS_DEV_EXTENT_KEY)
a2de733c
AJ		 2650 break;
2651
2652 if (found_key.offset >= end)
2653 break;
2654
2655 if (found_key.offset < key.offset)
2656 break;
2657
2658 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
2659 length = btrfs_dev_extent_length(l, dev_extent);
2660
2661 if (found_key.offset + length <= start) {
2662 key.offset = found_key.offset + length;
71267333 2663 btrfs_release_path(path);
a2de733c
AJ		 2664 continue;
2665 }
2666
2667 chunk_tree = btrfs_dev_extent_chunk_tree(l, dev_extent);
2668 chunk_objectid = btrfs_dev_extent_chunk_objectid(l, dev_extent);
2669 chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);
2670
2671 /*
2672 * get a reference on the corresponding block group to prevent
2673 * the chunk from going away while we scrub it
2674 */
2675 cache = btrfs_lookup_block_group(fs_info, chunk_offset);
2676 if (!cache) {
2677 ret = -ENOENT;
8c51032f 2678 break;
a2de733c 2679 }
ff023aac
SB		 2680 dev_replace->cursor_right = found_key.offset + length;
2681 dev_replace->cursor_left = found_key.offset;
2682 dev_replace->item_needs_writeback = 1;
a36cf8b8 2683 ret = scrub_chunk(sctx, scrub_dev, chunk_tree, chunk_objectid,
ff023aac
SB		 2684 chunk_offset, length, found_key.offset,
2685 is_dev_replace);
2686
2687 /*
2688 * flush, submit all pending read and write bios, afterwards
2689 * wait for them.
2690 * Note that in the dev replace case, a read request causes
2691 * write requests that are submitted in the read completion
2692 * worker. Therefore in the current situation, it is required
2693 * that all write requests are flushed, so that all read and
2694 * write requests are really completed when bios_in_flight
2695 * changes to 0.
2696 */
2697 atomic_set(&sctx->wr_ctx.flush_all_writes, 1);
2698 scrub_submit(sctx);
2699 mutex_lock(&sctx->wr_ctx.wr_lock);
2700 scrub_wr_submit(sctx);
2701 mutex_unlock(&sctx->wr_ctx.wr_lock);
2702
2703 wait_event(sctx->list_wait,
2704 atomic_read(&sctx->bios_in_flight) == 0);
2705 atomic_set(&sctx->wr_ctx.flush_all_writes, 0);
2706 atomic_inc(&fs_info->scrubs_paused);
2707 wake_up(&fs_info->scrub_pause_wait);
2708 wait_event(sctx->list_wait,
2709 atomic_read(&sctx->workers_pending) == 0);
2710
2711 mutex_lock(&fs_info->scrub_lock);
2712 while (atomic_read(&fs_info->scrub_pause_req)) {
2713 mutex_unlock(&fs_info->scrub_lock);
2714 wait_event(fs_info->scrub_pause_wait,
2715 atomic_read(&fs_info->scrub_pause_req) == 0);
2716 mutex_lock(&fs_info->scrub_lock);
2717 }
2718 atomic_dec(&fs_info->scrubs_paused);
2719 mutex_unlock(&fs_info->scrub_lock);
2720 wake_up(&fs_info->scrub_pause_wait);
2721
a2de733c
AJ		 2722 btrfs_put_block_group(cache);
2723 if (ret)
2724 break;
af1be4f8
SB		 2725 if (is_dev_replace &&
2726 atomic64_read(&dev_replace->num_write_errors) > 0) {
ff023aac
SB		 2727 ret = -EIO;
2728 break;
2729 }
2730 if (sctx->stat.malloc_errors > 0) {
2731 ret = -ENOMEM;
2732 break;
2733 }
a2de733c 2734
539f358a
ID		 2735 dev_replace->cursor_left = dev_replace->cursor_right;
2736 dev_replace->item_needs_writeback = 1;
2737
a2de733c 2738 key.offset = found_key.offset + length;
71267333 2739 btrfs_release_path(path);
a2de733c
AJ		 2740 }
2741
a2de733c 2742 btrfs_free_path(path);
8c51032f
AJ		 2743
2744 /*
2745 * ret can still be 1 from search_slot or next_leaf,
2746 * that's not an error
2747 */
2748 return ret < 0 ? ret : 0;
a2de733c
AJ		 2749}
2750
a36cf8b8
SB		 2751static noinline_for_stack int scrub_supers(struct scrub_ctx *sctx,
2752 struct btrfs_device *scrub_dev)
a2de733c
AJ		 2753{
2754 int i;
2755 u64 bytenr;
2756 u64 gen;
2757 int ret;
a36cf8b8 2758 struct btrfs_root *root = sctx->dev_root;
a2de733c 2759
87533c47 2760 if (test_bit(BTRFS_FS_STATE_ERROR, &root->fs_info->fs_state))
79787eaa
JM		 2761 return -EIO;
2762
a2de733c
AJ		 2763 gen = root->fs_info->last_trans_committed;
2764
2765 for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
2766 bytenr = btrfs_sb_offset(i);
a36cf8b8 2767 if (bytenr + BTRFS_SUPER_INFO_SIZE > scrub_dev->total_bytes)
a2de733c
AJ		 2768 break;
2769
d9d181c1 2770 ret = scrub_pages(sctx, bytenr, BTRFS_SUPER_INFO_SIZE, bytenr,
a36cf8b8 2771 scrub_dev, BTRFS_EXTENT_FLAG_SUPER, gen, i,
ff023aac 2772 NULL, 1, bytenr);
a2de733c
AJ		 2773 if (ret)
2774 return ret;
2775 }
b6bfebc1 2776 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
a2de733c
AJ		 2777
2778 return 0;
2779}
2780
2781/*
2782 * get a reference count on fs_info->scrub_workers. start worker if necessary
2783 */
ff023aac
SB		 2784static noinline_for_stack int scrub_workers_get(struct btrfs_fs_info *fs_info,
2785 int is_dev_replace)
a2de733c 2786{
0dc3b84a 2787 int ret = 0;
a2de733c 2788
632dd772 2789 if (fs_info->scrub_workers_refcnt == 0) {
ff023aac
SB		 2790 if (is_dev_replace)
2791 btrfs_init_workers(&fs_info->scrub_workers, "scrub", 1,
2792 &fs_info->generic_worker);
2793 else
2794 btrfs_init_workers(&fs_info->scrub_workers, "scrub",
2795 fs_info->thread_pool_size,
2796 &fs_info->generic_worker);
632dd772 2797 fs_info->scrub_workers.idle_thresh = 4;
0dc3b84a
JB		 2798 ret = btrfs_start_workers(&fs_info->scrub_workers);
2799 if (ret)
2800 goto out;
ff023aac
SB		 2801 btrfs_init_workers(&fs_info->scrub_wr_completion_workers,
2802 "scrubwrc",
2803 fs_info->thread_pool_size,
2804 &fs_info->generic_worker);
2805 fs_info->scrub_wr_completion_workers.idle_thresh = 2;
2806 ret = btrfs_start_workers(
2807 &fs_info->scrub_wr_completion_workers);
2808 if (ret)
2809 goto out;
2810 btrfs_init_workers(&fs_info->scrub_nocow_workers, "scrubnc", 1,
2811 &fs_info->generic_worker);
2812 ret = btrfs_start_workers(&fs_info->scrub_nocow_workers);
2813 if (ret)
2814 goto out;
632dd772 2815 }
a2de733c 2816 ++fs_info->scrub_workers_refcnt;
0dc3b84a 2817out:
0dc3b84a 2818 return ret;
a2de733c
AJ		 2819}
2820
aa1b8cd4 2821static noinline_for_stack void scrub_workers_put(struct btrfs_fs_info *fs_info)
a2de733c 2822{
ff023aac 2823 if (--fs_info->scrub_workers_refcnt == 0) {
a2de733c 2824 btrfs_stop_workers(&fs_info->scrub_workers);
ff023aac
SB		 2825 btrfs_stop_workers(&fs_info->scrub_wr_completion_workers);
2826 btrfs_stop_workers(&fs_info->scrub_nocow_workers);
2827 }
a2de733c 2828 WARN_ON(fs_info->scrub_workers_refcnt < 0);
a2de733c
AJ		 2829}
2830
aa1b8cd4
SB		 2831int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start,
2832 u64 end, struct btrfs_scrub_progress *progress,
63a212ab 2833 int readonly, int is_dev_replace)
a2de733c 2834{
d9d181c1 2835 struct scrub_ctx *sctx;
a2de733c
AJ		 2836 int ret;
2837 struct btrfs_device *dev;
2838
aa1b8cd4 2839 if (btrfs_fs_closing(fs_info))
a2de733c
AJ		 2840 return -EINVAL;
2841
2842 /*
2843 * check some assumptions
2844 */
aa1b8cd4 2845 if (fs_info->chunk_root->nodesize != fs_info->chunk_root->leafsize) {
b5d67f64
SB		 2846 printk(KERN_ERR
2847 "btrfs_scrub: size assumption nodesize == leafsize (%d == %d) fails\n",
aa1b8cd4
SB		 2848 fs_info->chunk_root->nodesize,
2849 fs_info->chunk_root->leafsize);
b5d67f64
SB		 2850 return -EINVAL;
2851 }
2852
aa1b8cd4 2853 if (fs_info->chunk_root->nodesize > BTRFS_STRIPE_LEN) {
b5d67f64
SB		 2854 /*
2855 * in this case scrub is unable to calculate the checksum
2856 * the way scrub is implemented. Do not handle this
2857 * situation at all because it won't ever happen.
2858 */
2859 printk(KERN_ERR
2860 "btrfs_scrub: size assumption nodesize <= BTRFS_STRIPE_LEN (%d <= %d) fails\n",
aa1b8cd4 2861 fs_info->chunk_root->nodesize, BTRFS_STRIPE_LEN);
b5d67f64
SB		 2862 return -EINVAL;
2863 }
2864
aa1b8cd4 2865 if (fs_info->chunk_root->sectorsize != PAGE_SIZE) {
b5d67f64
SB		 2866 /* not supported for data w/o checksums */
2867 printk(KERN_ERR
27f9f023
GU		 2868 "btrfs_scrub: size assumption sectorsize != PAGE_SIZE (%d != %lu) fails\n",
2869 fs_info->chunk_root->sectorsize, PAGE_SIZE);
a2de733c
AJ		 2870 return -EINVAL;
2871 }
2872
7a9e9987
SB		 2873 if (fs_info->chunk_root->nodesize >
2874 PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK ||
2875 fs_info->chunk_root->sectorsize >
2876 PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK) {
2877 /*
2878 * would exhaust the array bounds of pagev member in
2879 * struct scrub_block
2880 */
2881 pr_err("btrfs_scrub: size assumption nodesize and sectorsize <= SCRUB_MAX_PAGES_PER_BLOCK (%d <= %d && %d <= %d) fails\n",
2882 fs_info->chunk_root->nodesize,
2883 SCRUB_MAX_PAGES_PER_BLOCK,
2884 fs_info->chunk_root->sectorsize,
2885 SCRUB_MAX_PAGES_PER_BLOCK);
2886 return -EINVAL;
2887 }
2888
a2de733c 2889
aa1b8cd4
SB		 2890 mutex_lock(&fs_info->fs_devices->device_list_mutex);
2891 dev = btrfs_find_device(fs_info, devid, NULL, NULL);
63a212ab 2892 if (!dev || (dev->missing && !is_dev_replace)) {
aa1b8cd4 2893 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
a2de733c
AJ		 2894 return -ENODEV;
2895 }
a2de733c 2896
3b7a016f 2897 mutex_lock(&fs_info->scrub_lock);
63a212ab 2898 if (!dev->in_fs_metadata || dev->is_tgtdev_for_dev_replace) {
a2de733c 2899 mutex_unlock(&fs_info->scrub_lock);
aa1b8cd4 2900 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
aa1b8cd4 2901 return -EIO;
a2de733c
AJ		 2902 }
2903
8dabb742
SB		 2904 btrfs_dev_replace_lock(&fs_info->dev_replace);
2905 if (dev->scrub_device ||
2906 (!is_dev_replace &&
2907 btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))) {
2908 btrfs_dev_replace_unlock(&fs_info->dev_replace);
a2de733c 2909 mutex_unlock(&fs_info->scrub_lock);
aa1b8cd4 2910 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
a2de733c
AJ		 2911 return -EINPROGRESS;
2912 }
8dabb742 2913 btrfs_dev_replace_unlock(&fs_info->dev_replace);
3b7a016f
WS		 2914
2915 ret = scrub_workers_get(fs_info, is_dev_replace);
2916 if (ret) {
2917 mutex_unlock(&fs_info->scrub_lock);
2918 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2919 return ret;
2920 }
2921
63a212ab 2922 sctx = scrub_setup_ctx(dev, is_dev_replace);
d9d181c1 2923 if (IS_ERR(sctx)) {
a2de733c 2924 mutex_unlock(&fs_info->scrub_lock);
aa1b8cd4
SB		 2925 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2926 scrub_workers_put(fs_info);
d9d181c1 2927 return PTR_ERR(sctx);
a2de733c 2928 }
d9d181c1
SB		 2929 sctx->readonly = readonly;
2930 dev->scrub_device = sctx;
a2de733c
AJ		 2931
2932 atomic_inc(&fs_info->scrubs_running);
2933 mutex_unlock(&fs_info->scrub_lock);
a2de733c 2934
ff023aac 2935 if (!is_dev_replace) {
9b011adf
WS		 2936 /*
2937 * by holding device list mutex, we can
2938 * kick off writing super in log tree sync.
2939 */
ff023aac 2940 ret = scrub_supers(sctx, dev);
ff023aac 2941 }
9b011adf 2942 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
a2de733c
AJ		 2943
2944 if (!ret)
ff023aac
SB		 2945 ret = scrub_enumerate_chunks(sctx, dev, start, end,
2946 is_dev_replace);
a2de733c 2947
b6bfebc1 2948 wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
a2de733c
AJ		 2949 atomic_dec(&fs_info->scrubs_running);
2950 wake_up(&fs_info->scrub_pause_wait);
2951
b6bfebc1 2952 wait_event(sctx->list_wait, atomic_read(&sctx->workers_pending) == 0);
0ef8e451 2953
a2de733c 2954 if (progress)
d9d181c1 2955 memcpy(progress, &sctx->stat, sizeof(*progress));
a2de733c
AJ		 2956
2957 mutex_lock(&fs_info->scrub_lock);
2958 dev->scrub_device = NULL;
3b7a016f 2959 scrub_workers_put(fs_info);
a2de733c
AJ		 2960 mutex_unlock(&fs_info->scrub_lock);
2961
d9d181c1 2962 scrub_free_ctx(sctx);
a2de733c
AJ		 2963
2964 return ret;
2965}
2966
143bede5 2967void btrfs_scrub_pause(struct btrfs_root *root)
a2de733c
AJ		 2968{
2969 struct btrfs_fs_info *fs_info = root->fs_info;
2970
2971 mutex_lock(&fs_info->scrub_lock);
2972 atomic_inc(&fs_info->scrub_pause_req);
2973 while (atomic_read(&fs_info->scrubs_paused) !=
2974 atomic_read(&fs_info->scrubs_running)) {
2975 mutex_unlock(&fs_info->scrub_lock);
2976 wait_event(fs_info->scrub_pause_wait,
2977 atomic_read(&fs_info->scrubs_paused) ==
2978 atomic_read(&fs_info->scrubs_running));
2979 mutex_lock(&fs_info->scrub_lock);
2980 }
2981 mutex_unlock(&fs_info->scrub_lock);
a2de733c
AJ		 2982}
2983
143bede5 2984void btrfs_scrub_continue(struct btrfs_root *root)
a2de733c
AJ		 2985{
2986 struct btrfs_fs_info *fs_info = root->fs_info;
2987
2988 atomic_dec(&fs_info->scrub_pause_req);
2989 wake_up(&fs_info->scrub_pause_wait);
a2de733c
AJ		 2990}
2991
aa1b8cd4 2992int btrfs_scrub_cancel(struct btrfs_fs_info *fs_info)
a2de733c 2993{
a2de733c
AJ		 2994 mutex_lock(&fs_info->scrub_lock);
2995 if (!atomic_read(&fs_info->scrubs_running)) {
2996 mutex_unlock(&fs_info->scrub_lock);
2997 return -ENOTCONN;
2998 }
2999
3000 atomic_inc(&fs_info->scrub_cancel_req);
3001 while (atomic_read(&fs_info->scrubs_running)) {
3002 mutex_unlock(&fs_info->scrub_lock);
3003 wait_event(fs_info->scrub_pause_wait,
3004 atomic_read(&fs_info->scrubs_running) == 0);
3005 mutex_lock(&fs_info->scrub_lock);
3006 }
3007 atomic_dec(&fs_info->scrub_cancel_req);
3008 mutex_unlock(&fs_info->scrub_lock);
3009
3010 return 0;
3011}
3012
aa1b8cd4
SB		 3013int btrfs_scrub_cancel_dev(struct btrfs_fs_info *fs_info,
3014 struct btrfs_device *dev)
49b25e05 3015{
d9d181c1 3016 struct scrub_ctx *sctx;
a2de733c
AJ		 3017
3018 mutex_lock(&fs_info->scrub_lock);
d9d181c1
SB		 3019 sctx = dev->scrub_device;
3020 if (!sctx) {
a2de733c
AJ		 3021 mutex_unlock(&fs_info->scrub_lock);
3022 return -ENOTCONN;
3023 }
d9d181c1 3024 atomic_inc(&sctx->cancel_req);
a2de733c
AJ		 3025 while (dev->scrub_device) {
3026 mutex_unlock(&fs_info->scrub_lock);
3027 wait_event(fs_info->scrub_pause_wait,
3028 dev->scrub_device == NULL);
3029 mutex_lock(&fs_info->scrub_lock);
3030 }
3031 mutex_unlock(&fs_info->scrub_lock);
3032
3033 return 0;
3034}
1623edeb 3035
a2de733c
AJ		 3036int btrfs_scrub_progress(struct btrfs_root *root, u64 devid,
3037 struct btrfs_scrub_progress *progress)
3038{
3039 struct btrfs_device *dev;
d9d181c1 3040 struct scrub_ctx *sctx = NULL;
a2de733c
AJ		 3041
3042 mutex_lock(&root->fs_info->fs_devices->device_list_mutex);
aa1b8cd4 3043 dev = btrfs_find_device(root->fs_info, devid, NULL, NULL);
a2de733c 3044 if (dev)
d9d181c1
SB		 3045 sctx = dev->scrub_device;
3046 if (sctx)
3047 memcpy(progress, &sctx->stat, sizeof(*progress));
a2de733c
AJ		 3048 mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
3049
d9d181c1 3050 return dev ? (sctx ? 0 : -ENOTCONN) : -ENODEV;
a2de733c 3051}
ff023aac
SB		 3052
3053static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
3054 u64 extent_logical, u64 extent_len,
3055 u64 *extent_physical,
3056 struct btrfs_device **extent_dev,
3057 int *extent_mirror_num)
3058{
3059 u64 mapped_length;
3060 struct btrfs_bio *bbio = NULL;
3061 int ret;
3062
3063 mapped_length = extent_len;
3064 ret = btrfs_map_block(fs_info, READ, extent_logical,
3065 &mapped_length, &bbio, 0);
3066 if (ret || !bbio || mapped_length < extent_len ||
3067 !bbio->stripes[0].dev->bdev) {
3068 kfree(bbio);
3069 return;
3070 }
3071
3072 *extent_physical = bbio->stripes[0].physical;
3073 *extent_mirror_num = bbio->mirror_num;
3074 *extent_dev = bbio->stripes[0].dev;
3075 kfree(bbio);
3076}
3077
3078static int scrub_setup_wr_ctx(struct scrub_ctx *sctx,
3079 struct scrub_wr_ctx *wr_ctx,
3080 struct btrfs_fs_info *fs_info,
3081 struct btrfs_device *dev,
3082 int is_dev_replace)
3083{
3084 WARN_ON(wr_ctx->wr_curr_bio != NULL);
3085
3086 mutex_init(&wr_ctx->wr_lock);
3087 wr_ctx->wr_curr_bio = NULL;
3088 if (!is_dev_replace)
3089 return 0;
3090
3091 WARN_ON(!dev->bdev);
3092 wr_ctx->pages_per_wr_bio = min_t(int, SCRUB_PAGES_PER_WR_BIO,
3093 bio_get_nr_vecs(dev->bdev));
3094 wr_ctx->tgtdev = dev;
3095 atomic_set(&wr_ctx->flush_all_writes, 0);
3096 return 0;
3097}
3098
3099static void scrub_free_wr_ctx(struct scrub_wr_ctx *wr_ctx)
3100{
3101 mutex_lock(&wr_ctx->wr_lock);
3102 kfree(wr_ctx->wr_curr_bio);
3103 wr_ctx->wr_curr_bio = NULL;
3104 mutex_unlock(&wr_ctx->wr_lock);
3105}
3106
3107static int copy_nocow_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
3108 int mirror_num, u64 physical_for_dev_replace)
3109{
3110 struct scrub_copy_nocow_ctx *nocow_ctx;
3111 struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
3112
3113 nocow_ctx = kzalloc(sizeof(*nocow_ctx), GFP_NOFS);
3114 if (!nocow_ctx) {
3115 spin_lock(&sctx->stat_lock);
3116 sctx->stat.malloc_errors++;
3117 spin_unlock(&sctx->stat_lock);
3118 return -ENOMEM;
3119 }
3120
3121 scrub_pending_trans_workers_inc(sctx);
3122
3123 nocow_ctx->sctx = sctx;
3124 nocow_ctx->logical = logical;
3125 nocow_ctx->len = len;
3126 nocow_ctx->mirror_num = mirror_num;
3127 nocow_ctx->physical_for_dev_replace = physical_for_dev_replace;
3128 nocow_ctx->work.func = copy_nocow_pages_worker;
652f25a2 3129 INIT_LIST_HEAD(&nocow_ctx->inodes);
ff023aac
SB		 3130 btrfs_queue_worker(&fs_info->scrub_nocow_workers,
3131 &nocow_ctx->work);
3132
3133 return 0;
3134}
3135
652f25a2
JB		 3136static int record_inode_for_nocow(u64 inum, u64 offset, u64 root, void *ctx)
3137{
3138 struct scrub_copy_nocow_ctx *nocow_ctx = ctx;
3139 struct scrub_nocow_inode *nocow_inode;
3140
3141 nocow_inode = kzalloc(sizeof(*nocow_inode), GFP_NOFS);
3142 if (!nocow_inode)
3143 return -ENOMEM;
3144 nocow_inode->inum = inum;
3145 nocow_inode->offset = offset;
3146 nocow_inode->root = root;
3147 list_add_tail(&nocow_inode->list, &nocow_ctx->inodes);
3148 return 0;
3149}
3150
3151#define COPY_COMPLETE 1
3152
ff023aac
SB		 3153static void copy_nocow_pages_worker(struct btrfs_work *work)
3154{
3155 struct scrub_copy_nocow_ctx *nocow_ctx =
3156 container_of(work, struct scrub_copy_nocow_ctx, work);
3157 struct scrub_ctx *sctx = nocow_ctx->sctx;
3158 u64 logical = nocow_ctx->logical;
3159 u64 len = nocow_ctx->len;
3160 int mirror_num = nocow_ctx->mirror_num;
3161 u64 physical_for_dev_replace = nocow_ctx->physical_for_dev_replace;
3162 int ret;
3163 struct btrfs_trans_handle *trans = NULL;
3164 struct btrfs_fs_info *fs_info;
3165 struct btrfs_path *path;
3166 struct btrfs_root *root;
3167 int not_written = 0;
3168
3169 fs_info = sctx->dev_root->fs_info;
3170 root = fs_info->extent_root;
3171
3172 path = btrfs_alloc_path();
3173 if (!path) {
3174 spin_lock(&sctx->stat_lock);
3175 sctx->stat.malloc_errors++;
3176 spin_unlock(&sctx->stat_lock);
3177 not_written = 1;
3178 goto out;
3179 }
3180
3181 trans = btrfs_join_transaction(root);
3182 if (IS_ERR(trans)) {
3183 not_written = 1;
3184 goto out;
3185 }
3186
3187 ret = iterate_inodes_from_logical(logical, fs_info, path,
652f25a2 3188 record_inode_for_nocow, nocow_ctx);
ff023aac 3189 if (ret != 0 && ret != -ENOENT) {
118a0a25
GU		 3190 pr_warn("iterate_inodes_from_logical() failed: log %llu, phys %llu, len %llu, mir %u, ret %d\n",
3191 logical, physical_for_dev_replace, len, mirror_num,
3192 ret);
ff023aac
SB		 3193 not_written = 1;
3194 goto out;
3195 }
3196
652f25a2
JB		 3197 btrfs_end_transaction(trans, root);
3198 trans = NULL;
3199 while (!list_empty(&nocow_ctx->inodes)) {
3200 struct scrub_nocow_inode *entry;
3201 entry = list_first_entry(&nocow_ctx->inodes,
3202 struct scrub_nocow_inode,
3203 list);
3204 list_del_init(&entry->list);
3205 ret = copy_nocow_pages_for_inode(entry->inum, entry->offset,
3206 entry->root, nocow_ctx);
3207 kfree(entry);
3208 if (ret == COPY_COMPLETE) {
3209 ret = 0;
3210 break;
3211 } else if (ret) {
3212 break;
3213 }
3214 }
ff023aac 3215out:
652f25a2
JB		 3216 while (!list_empty(&nocow_ctx->inodes)) {
3217 struct scrub_nocow_inode *entry;
3218 entry = list_first_entry(&nocow_ctx->inodes,
3219 struct scrub_nocow_inode,
3220 list);
3221 list_del_init(&entry->list);
3222 kfree(entry);
3223 }
ff023aac
SB		 3224 if (trans && !IS_ERR(trans))
3225 btrfs_end_transaction(trans, root);
3226 if (not_written)
3227 btrfs_dev_replace_stats_inc(&fs_info->dev_replace.
3228 num_uncorrectable_read_errors);
3229
3230 btrfs_free_path(path);
3231 kfree(nocow_ctx);
3232
3233 scrub_pending_trans_workers_dec(sctx);
3234}
3235
652f25a2
JB		 3236static int copy_nocow_pages_for_inode(u64 inum, u64 offset, u64 root,
3237 struct scrub_copy_nocow_ctx *nocow_ctx)
ff023aac 3238{
826aa0a8 3239 struct btrfs_fs_info *fs_info = nocow_ctx->sctx->dev_root->fs_info;
ff023aac 3240 struct btrfs_key key;
826aa0a8
MX		 3241 struct inode *inode;
3242 struct page *page;
ff023aac 3243 struct btrfs_root *local_root;
652f25a2
JB		 3244 struct btrfs_ordered_extent *ordered;
3245 struct extent_map *em;
3246 struct extent_state *cached_state = NULL;
3247 struct extent_io_tree *io_tree;
ff023aac 3248 u64 physical_for_dev_replace;
652f25a2
JB		 3249 u64 len = nocow_ctx->len;
3250 u64 lockstart = offset, lockend = offset + len - 1;
826aa0a8 3251 unsigned long index;
6f1c3605 3252 int srcu_index;
652f25a2
JB		 3253 int ret = 0;
3254 int err = 0;
ff023aac
SB		 3255
3256 key.objectid = root;
3257 key.type = BTRFS_ROOT_ITEM_KEY;
3258 key.offset = (u64)-1;
6f1c3605
LB		 3259
3260 srcu_index = srcu_read_lock(&fs_info->subvol_srcu);
3261
ff023aac 3262 local_root = btrfs_read_fs_root_no_name(fs_info, &key);
6f1c3605
LB		 3263 if (IS_ERR(local_root)) {
3264 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
ff023aac 3265 return PTR_ERR(local_root);
6f1c3605 3266 }
ff023aac
SB		 3267
3268 key.type = BTRFS_INODE_ITEM_KEY;
3269 key.objectid = inum;
3270 key.offset = 0;
3271 inode = btrfs_iget(fs_info->sb, &key, local_root, NULL);
6f1c3605 3272 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
ff023aac
SB		 3273 if (IS_ERR(inode))
3274 return PTR_ERR(inode);
3275
edd1400b
MX		 3276 /* Avoid truncate/dio/punch hole.. */
3277 mutex_lock(&inode->i_mutex);
3278 inode_dio_wait(inode);
3279
ff023aac 3280 physical_for_dev_replace = nocow_ctx->physical_for_dev_replace;
652f25a2
JB		 3281 io_tree = &BTRFS_I(inode)->io_tree;
3282
3283 lock_extent_bits(io_tree, lockstart, lockend, 0, &cached_state);
3284 ordered = btrfs_lookup_ordered_range(inode, lockstart, len);
3285 if (ordered) {
3286 btrfs_put_ordered_extent(ordered);
3287 goto out_unlock;
3288 }
3289
3290 em = btrfs_get_extent(inode, NULL, 0, lockstart, len, 0);
3291 if (IS_ERR(em)) {
3292 ret = PTR_ERR(em);
3293 goto out_unlock;
3294 }
3295
3296 /*
3297 * This extent does not actually cover the logical extent anymore,
3298 * move on to the next inode.
3299 */
3300 if (em->block_start > nocow_ctx->logical ||
3301 em->block_start + em->block_len < nocow_ctx->logical + len) {
3302 free_extent_map(em);
3303 goto out_unlock;
3304 }
3305 free_extent_map(em);
3306
ff023aac 3307 while (len >= PAGE_CACHE_SIZE) {
ff023aac 3308 index = offset >> PAGE_CACHE_SHIFT;
edd1400b 3309again:
ff023aac
SB		 3310 page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
3311 if (!page) {
3312 pr_err("find_or_create_page() failed\n");
3313 ret = -ENOMEM;
826aa0a8 3314 goto out;
ff023aac
SB		 3315 }
3316
3317 if (PageUptodate(page)) {
3318 if (PageDirty(page))
3319 goto next_page;
3320 } else {
3321 ClearPageError(page);
652f25a2
JB		 3322 err = extent_read_full_page_nolock(io_tree, page,
3323 btrfs_get_extent,
3324 nocow_ctx->mirror_num);
826aa0a8
MX		 3325 if (err) {
3326 ret = err;
ff023aac
SB		 3327 goto next_page;
3328 }
edd1400b 3329
26b25891 3330 lock_page(page);
edd1400b
MX		 3331 /*
3332 * If the page has been remove from the page cache,
3333 * the data on it is meaningless, because it may be
3334 * old one, the new data may be written into the new
3335 * page in the page cache.
3336 */
3337 if (page->mapping != inode->i_mapping) {
652f25a2 3338 unlock_page(page);
edd1400b
MX		 3339 page_cache_release(page);
3340 goto again;
3341 }
ff023aac
SB		 3342 if (!PageUptodate(page)) {
3343 ret = -EIO;
3344 goto next_page;
3345 }
3346 }
826aa0a8
MX		 3347 err = write_page_nocow(nocow_ctx->sctx,
3348 physical_for_dev_replace, page);
3349 if (err)
3350 ret = err;
ff023aac 3351next_page:
826aa0a8
MX		 3352 unlock_page(page);
3353 page_cache_release(page);
3354
3355 if (ret)
3356 break;
3357
ff023aac
SB		 3358 offset += PAGE_CACHE_SIZE;
3359 physical_for_dev_replace += PAGE_CACHE_SIZE;
3360 len -= PAGE_CACHE_SIZE;
3361 }
652f25a2
JB		 3362 ret = COPY_COMPLETE;
3363out_unlock:
3364 unlock_extent_cached(io_tree, lockstart, lockend, &cached_state,
3365 GFP_NOFS);
826aa0a8 3366out:
edd1400b 3367 mutex_unlock(&inode->i_mutex);
826aa0a8 3368 iput(inode);
ff023aac
SB		 3369 return ret;
3370}
3371
3372static int write_page_nocow(struct scrub_ctx *sctx,
3373 u64 physical_for_dev_replace, struct page *page)
3374{
3375 struct bio *bio;
3376 struct btrfs_device *dev;
3377 int ret;
3378 DECLARE_COMPLETION_ONSTACK(compl);
3379
3380 dev = sctx->wr_ctx.tgtdev;
3381 if (!dev)
3382 return -EIO;
3383 if (!dev->bdev) {
3384 printk_ratelimited(KERN_WARNING
3385 "btrfs: scrub write_page_nocow(bdev == NULL) is unexpected!\n");
3386 return -EIO;
3387 }
9be3395b 3388 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
ff023aac
SB		 3389 if (!bio) {
3390 spin_lock(&sctx->stat_lock);
3391 sctx->stat.malloc_errors++;
3392 spin_unlock(&sctx->stat_lock);
3393 return -ENOMEM;
3394 }
3395 bio->bi_private = &compl;
3396 bio->bi_end_io = scrub_complete_bio_end_io;
3397 bio->bi_size = 0;
3398 bio->bi_sector = physical_for_dev_replace >> 9;
3399 bio->bi_bdev = dev->bdev;
3400 ret = bio_add_page(bio, page, PAGE_CACHE_SIZE, 0);
3401 if (ret != PAGE_CACHE_SIZE) {
3402leave_with_eio:
3403 bio_put(bio);
3404 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
3405 return -EIO;
3406 }
3407 btrfsic_submit_bio(WRITE_SYNC, bio);
3408 wait_for_completion(&compl);
3409
3410 if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
3411 goto leave_with_eio;
3412
3413 bio_put(bio);
3414 return 0;
3415}
Linux kernel - Deliverables
This page took 0.353658 seconds and 5 git commands to generate.