projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
f2fs: introduce f2fs_journal struct to wrap journal info
[deliverable/linux.git] / fs / f2fs / segment.c
1 /*
2 * fs/f2fs/segment.c
3 *
4 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
5 * http://www.samsung.com/
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11 #include <linux/fs.h>
12 #include <linux/f2fs_fs.h>
13 #include <linux/bio.h>
14 #include <linux/blkdev.h>
15 #include <linux/prefetch.h>
16 #include <linux/kthread.h>
17 #include <linux/swap.h>
18 #include <linux/timer.h>
19
20 #include "f2fs.h"
21 #include "segment.h"
22 #include "node.h"
23 #include "trace.h"
24 #include <trace/events/f2fs.h>
25
26 #define __reverse_ffz(x) __reverse_ffs(~(x))
27
28 static struct kmem_cache *discard_entry_slab;
29 static struct kmem_cache *sit_entry_set_slab;
30 static struct kmem_cache *inmem_entry_slab;
31
32 static unsigned long __reverse_ulong(unsigned char *str)
33 {
34 unsigned long tmp = 0;
35 int shift = 24, idx = 0;
36
37 #if BITS_PER_LONG == 64
38 shift = 56;
39 #endif
40 while (shift >= 0) {
41 tmp |= (unsigned long)str[idx++] << shift;
42 shift -= BITS_PER_BYTE;
43 }
44 return tmp;
45 }
46
47 /*
48 * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since
49 * MSB and LSB are reversed in a byte by f2fs_set_bit.
50 */
51 static inline unsigned long __reverse_ffs(unsigned long word)
52 {
53 int num = 0;
54
55 #if BITS_PER_LONG == 64
56 if ((word & 0xffffffff00000000UL) == 0)
57 num += 32;
58 else
59 word >>= 32;
60 #endif
61 if ((word & 0xffff0000) == 0)
62 num += 16;
63 else
64 word >>= 16;
65
66 if ((word & 0xff00) == 0)
67 num += 8;
68 else
69 word >>= 8;
70
71 if ((word & 0xf0) == 0)
72 num += 4;
73 else
74 word >>= 4;
75
76 if ((word & 0xc) == 0)
77 num += 2;
78 else
79 word >>= 2;
80
81 if ((word & 0x2) == 0)
82 num += 1;
83 return num;
84 }
85
86 /*
87 * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because
88 * f2fs_set_bit makes MSB and LSB reversed in a byte.
89 * @size must be integral times of unsigned long.
90 * Example:
91 * MSB <--> LSB
92 * f2fs_set_bit(0, bitmap) => 1000 0000
93 * f2fs_set_bit(7, bitmap) => 0000 0001
94 */
95 static unsigned long __find_rev_next_bit(const unsigned long *addr,
96 unsigned long size, unsigned long offset)
97 {
98 const unsigned long *p = addr + BIT_WORD(offset);
99 unsigned long result = size;
100 unsigned long tmp;
101
102 if (offset >= size)
103 return size;
104
105 size -= (offset & ~(BITS_PER_LONG - 1));
106 offset %= BITS_PER_LONG;
107
108 while (1) {
109 if (*p == 0)
110 goto pass;
111
112 tmp = __reverse_ulong((unsigned char *)p);
113
114 tmp &= ~0UL >> offset;
115 if (size < BITS_PER_LONG)
116 tmp &= (~0UL << (BITS_PER_LONG - size));
117 if (tmp)
118 goto found;
119 pass:
120 if (size <= BITS_PER_LONG)
121 break;
122 size -= BITS_PER_LONG;
123 offset = 0;
124 p++;
125 }
126 return result;
127 found:
128 return result - size + __reverse_ffs(tmp);
129 }
130
131 static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
132 unsigned long size, unsigned long offset)
133 {
134 const unsigned long *p = addr + BIT_WORD(offset);
135 unsigned long result = size;
136 unsigned long tmp;
137
138 if (offset >= size)
139 return size;
140
141 size -= (offset & ~(BITS_PER_LONG - 1));
142 offset %= BITS_PER_LONG;
143
144 while (1) {
145 if (*p == ~0UL)
146 goto pass;
147
148 tmp = __reverse_ulong((unsigned char *)p);
149
150 if (offset)
151 tmp |= ~0UL << (BITS_PER_LONG - offset);
152 if (size < BITS_PER_LONG)
153 tmp |= ~0UL >> size;
154 if (tmp != ~0UL)
155 goto found;
156 pass:
157 if (size <= BITS_PER_LONG)
158 break;
159 size -= BITS_PER_LONG;
160 offset = 0;
161 p++;
162 }
163 return result;
164 found:
165 return result - size + __reverse_ffz(tmp);
166 }
167
168 void register_inmem_page(struct inode *inode, struct page *page)
169 {
170 struct f2fs_inode_info *fi = F2FS_I(inode);
171 struct inmem_pages *new;
172
173 f2fs_trace_pid(page);
174
175 set_page_private(page, (unsigned long)ATOMIC_WRITTEN_PAGE);
176 SetPagePrivate(page);
177
178 new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS);
179
180 /* add atomic page indices to the list */
181 new->page = page;
182 INIT_LIST_HEAD(&new->list);
183
184 /* increase reference count with clean state */
185 mutex_lock(&fi->inmem_lock);
186 get_page(page);
187 list_add_tail(&new->list, &fi->inmem_pages);
188 inc_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
189 mutex_unlock(&fi->inmem_lock);
190
191 trace_f2fs_register_inmem_page(page, INMEM);
192 }
193
194 static int __revoke_inmem_pages(struct inode *inode,
195 struct list_head *head, bool drop, bool recover)
196 {
197 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
198 struct inmem_pages *cur, *tmp;
199 int err = 0;
200
201 list_for_each_entry_safe(cur, tmp, head, list) {
202 struct page *page = cur->page;
203
204 if (drop)
205 trace_f2fs_commit_inmem_page(page, INMEM_DROP);
206
207 lock_page(page);
208
209 if (recover) {
210 struct dnode_of_data dn;
211 struct node_info ni;
212
213 trace_f2fs_commit_inmem_page(page, INMEM_REVOKE);
214
215 set_new_dnode(&dn, inode, NULL, NULL, 0);
216 if (get_dnode_of_data(&dn, page->index, LOOKUP_NODE)) {
217 err = -EAGAIN;
218 goto next;
219 }
220 get_node_info(sbi, dn.nid, &ni);
221 f2fs_replace_block(sbi, &dn, dn.data_blkaddr,
222 cur->old_addr, ni.version, true, true);
223 f2fs_put_dnode(&dn);
224 }
225 next:
226 ClearPageUptodate(page);
227 set_page_private(page, 0);
228 ClearPageUptodate(page);
229 f2fs_put_page(page, 1);
230
231 list_del(&cur->list);
232 kmem_cache_free(inmem_entry_slab, cur);
233 dec_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
234 }
235 return err;
236 }
237
238 void drop_inmem_pages(struct inode *inode)
239 {
240 struct f2fs_inode_info *fi = F2FS_I(inode);
241
242 mutex_lock(&fi->inmem_lock);
243 __revoke_inmem_pages(inode, &fi->inmem_pages, true, false);
244 mutex_unlock(&fi->inmem_lock);
245 }
246
247 static int __commit_inmem_pages(struct inode *inode,
248 struct list_head *revoke_list)
249 {
250 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
251 struct f2fs_inode_info *fi = F2FS_I(inode);
252 struct inmem_pages *cur, *tmp;
253 struct f2fs_io_info fio = {
254 .sbi = sbi,
255 .type = DATA,
256 .rw = WRITE_SYNC | REQ_PRIO,
257 .encrypted_page = NULL,
258 };
259 bool submit_bio = false;
260 int err = 0;
261
262 list_for_each_entry_safe(cur, tmp, &fi->inmem_pages, list) {
263 struct page *page = cur->page;
264
265 lock_page(page);
266 if (page->mapping == inode->i_mapping) {
267 trace_f2fs_commit_inmem_page(page, INMEM);
268
269 set_page_dirty(page);
270 f2fs_wait_on_page_writeback(page, DATA, true);
271 if (clear_page_dirty_for_io(page))
272 inode_dec_dirty_pages(inode);
273
274 fio.page = page;
275 err = do_write_data_page(&fio);
276 if (err) {
277 unlock_page(page);
278 break;
279 }
280
281 /* record old blkaddr for revoking */
282 cur->old_addr = fio.old_blkaddr;
283
284 clear_cold_data(page);
285 submit_bio = true;
286 }
287 unlock_page(page);
288 list_move_tail(&cur->list, revoke_list);
289 }
290
291 if (submit_bio)
292 f2fs_submit_merged_bio_cond(sbi, inode, NULL, 0, DATA, WRITE);
293
294 if (!err)
295 __revoke_inmem_pages(inode, revoke_list, false, false);
296
297 return err;
298 }
299
300 int commit_inmem_pages(struct inode *inode)
301 {
302 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
303 struct f2fs_inode_info *fi = F2FS_I(inode);
304 struct list_head revoke_list;
305 int err;
306
307 INIT_LIST_HEAD(&revoke_list);
308 f2fs_balance_fs(sbi, true);
309 f2fs_lock_op(sbi);
310
311 mutex_lock(&fi->inmem_lock);
312 err = __commit_inmem_pages(inode, &revoke_list);
313 if (err) {
314 int ret;
315 /*
316 * try to revoke all committed pages, but still we could fail
317 * due to no memory or other reason, if that happened, EAGAIN
318 * will be returned, which means in such case, transaction is
319 * already not integrity, caller should use journal to do the
320 * recovery or rewrite & commit last transaction. For other
321 * error number, revoking was done by filesystem itself.
322 */
323 ret = __revoke_inmem_pages(inode, &revoke_list, false, true);
324 if (ret)
325 err = ret;
326
327 /* drop all uncommitted pages */
328 __revoke_inmem_pages(inode, &fi->inmem_pages, true, false);
329 }
330 mutex_unlock(&fi->inmem_lock);
331
332 f2fs_unlock_op(sbi);
333 return err;
334 }
335
336 /*
337 * This function balances dirty node and dentry pages.
338 * In addition, it controls garbage collection.
339 */
340 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
341 {
342 if (!need)
343 return;
344 /*
345 * We should do GC or end up with checkpoint, if there are so many dirty
346 * dir/node pages without enough free segments.
347 */
348 if (has_not_enough_free_secs(sbi, 0)) {
349 mutex_lock(&sbi->gc_mutex);
350 f2fs_gc(sbi, false);
351 }
352 }
353
354 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi)
355 {
356 /* try to shrink extent cache when there is no enough memory */
357 if (!available_free_memory(sbi, EXTENT_CACHE))
358 f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER);
359
360 /* check the # of cached NAT entries */
361 if (!available_free_memory(sbi, NAT_ENTRIES))
362 try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);
363
364 if (!available_free_memory(sbi, FREE_NIDS))
365 try_to_free_nids(sbi, NAT_ENTRY_PER_BLOCK * FREE_NID_PAGES);
366
367 /* checkpoint is the only way to shrink partial cached entries */
368 if (!available_free_memory(sbi, NAT_ENTRIES) ||
369 !available_free_memory(sbi, INO_ENTRIES) ||
370 excess_prefree_segs(sbi) ||
371 excess_dirty_nats(sbi) ||
372 (is_idle(sbi) && f2fs_time_over(sbi, CP_TIME))) {
373 if (test_opt(sbi, DATA_FLUSH))
374 sync_dirty_inodes(sbi, FILE_INODE);
375 f2fs_sync_fs(sbi->sb, true);
376 stat_inc_bg_cp_count(sbi->stat_info);
377 }
378 }
379
380 static int issue_flush_thread(void *data)
381 {
382 struct f2fs_sb_info *sbi = data;
383 struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
384 wait_queue_head_t *q = &fcc->flush_wait_queue;
385 repeat:
386 if (kthread_should_stop())
387 return 0;
388
389 if (!llist_empty(&fcc->issue_list)) {
390 struct bio *bio;
391 struct flush_cmd *cmd, *next;
392 int ret;
393
394 bio = f2fs_bio_alloc(0);
395
396 fcc->dispatch_list = llist_del_all(&fcc->issue_list);
397 fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list);
398
399 bio->bi_bdev = sbi->sb->s_bdev;
400 ret = submit_bio_wait(WRITE_FLUSH, bio);
401
402 llist_for_each_entry_safe(cmd, next,
403 fcc->dispatch_list, llnode) {
404 cmd->ret = ret;
405 complete(&cmd->wait);
406 }
407 bio_put(bio);
408 fcc->dispatch_list = NULL;
409 }
410
411 wait_event_interruptible(*q,
412 kthread_should_stop() || !llist_empty(&fcc->issue_list));
413 goto repeat;
414 }
415
416 int f2fs_issue_flush(struct f2fs_sb_info *sbi)
417 {
418 struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
419 struct flush_cmd cmd;
420
421 trace_f2fs_issue_flush(sbi->sb, test_opt(sbi, NOBARRIER),
422 test_opt(sbi, FLUSH_MERGE));
423
424 if (test_opt(sbi, NOBARRIER))
425 return 0;
426
427 if (!test_opt(sbi, FLUSH_MERGE)) {
428 struct bio *bio = f2fs_bio_alloc(0);
429 int ret;
430
431 bio->bi_bdev = sbi->sb->s_bdev;
432 ret = submit_bio_wait(WRITE_FLUSH, bio);
433 bio_put(bio);
434 return ret;
435 }
436
437 init_completion(&cmd.wait);
438
439 llist_add(&cmd.llnode, &fcc->issue_list);
440
441 if (!fcc->dispatch_list)
442 wake_up(&fcc->flush_wait_queue);
443
444 wait_for_completion(&cmd.wait);
445
446 return cmd.ret;
447 }
448
449 int create_flush_cmd_control(struct f2fs_sb_info *sbi)
450 {
451 dev_t dev = sbi->sb->s_bdev->bd_dev;
452 struct flush_cmd_control *fcc;
453 int err = 0;
454
455 fcc = kzalloc(sizeof(struct flush_cmd_control), GFP_KERNEL);
456 if (!fcc)
457 return -ENOMEM;
458 init_waitqueue_head(&fcc->flush_wait_queue);
459 init_llist_head(&fcc->issue_list);
460 SM_I(sbi)->cmd_control_info = fcc;
461 fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,
462 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
463 if (IS_ERR(fcc->f2fs_issue_flush)) {
464 err = PTR_ERR(fcc->f2fs_issue_flush);
465 kfree(fcc);
466 SM_I(sbi)->cmd_control_info = NULL;
467 return err;
468 }
469
470 return err;
471 }
472
473 void destroy_flush_cmd_control(struct f2fs_sb_info *sbi)
474 {
475 struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
476
477 if (fcc && fcc->f2fs_issue_flush)
478 kthread_stop(fcc->f2fs_issue_flush);
479 kfree(fcc);
480 SM_I(sbi)->cmd_control_info = NULL;
481 }
482
483 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
484 enum dirty_type dirty_type)
485 {
486 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
487
488 /* need not be added */
489 if (IS_CURSEG(sbi, segno))
490 return;
491
492 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
493 dirty_i->nr_dirty[dirty_type]++;
494
495 if (dirty_type == DIRTY) {
496 struct seg_entry *sentry = get_seg_entry(sbi, segno);
497 enum dirty_type t = sentry->type;
498
499 if (unlikely(t >= DIRTY)) {
500 f2fs_bug_on(sbi, 1);
501 return;
502 }
503 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t]))
504 dirty_i->nr_dirty[t]++;
505 }
506 }
507
508 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
509 enum dirty_type dirty_type)
510 {
511 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
512
513 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
514 dirty_i->nr_dirty[dirty_type]--;
515
516 if (dirty_type == DIRTY) {
517 struct seg_entry *sentry = get_seg_entry(sbi, segno);
518 enum dirty_type t = sentry->type;
519
520 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
521 dirty_i->nr_dirty[t]--;
522
523 if (get_valid_blocks(sbi, segno, sbi->segs_per_sec) == 0)
524 clear_bit(GET_SECNO(sbi, segno),
525 dirty_i->victim_secmap);
526 }
527 }
528
529 /*
530 * Should not occur error such as -ENOMEM.
531 * Adding dirty entry into seglist is not critical operation.
532 * If a given segment is one of current working segments, it won't be added.
533 */
534 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
535 {
536 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
537 unsigned short valid_blocks;
538
539 if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))
540 return;
541
542 mutex_lock(&dirty_i->seglist_lock);
543
544 valid_blocks = get_valid_blocks(sbi, segno, 0);
545
546 if (valid_blocks == 0) {
547 __locate_dirty_segment(sbi, segno, PRE);
548 __remove_dirty_segment(sbi, segno, DIRTY);
549 } else if (valid_blocks < sbi->blocks_per_seg) {
550 __locate_dirty_segment(sbi, segno, DIRTY);
551 } else {
552 /* Recovery routine with SSR needs this */
553 __remove_dirty_segment(sbi, segno, DIRTY);
554 }
555
556 mutex_unlock(&dirty_i->seglist_lock);
557 }
558
559 static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
560 block_t blkstart, block_t blklen)
561 {
562 sector_t start = SECTOR_FROM_BLOCK(blkstart);
563 sector_t len = SECTOR_FROM_BLOCK(blklen);
564 struct seg_entry *se;
565 unsigned int offset;
566 block_t i;
567
568 for (i = blkstart; i < blkstart + blklen; i++) {
569 se = get_seg_entry(sbi, GET_SEGNO(sbi, i));
570 offset = GET_BLKOFF_FROM_SEG0(sbi, i);
571
572 if (!f2fs_test_and_set_bit(offset, se->discard_map))
573 sbi->discard_blks--;
574 }
575 trace_f2fs_issue_discard(sbi->sb, blkstart, blklen);
576 return blkdev_issue_discard(sbi->sb->s_bdev, start, len, GFP_NOFS, 0);
577 }
578
579 bool discard_next_dnode(struct f2fs_sb_info *sbi, block_t blkaddr)
580 {
581 int err = -EOPNOTSUPP;
582
583 if (test_opt(sbi, DISCARD)) {
584 struct seg_entry *se = get_seg_entry(sbi,
585 GET_SEGNO(sbi, blkaddr));
586 unsigned int offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
587
588 if (f2fs_test_bit(offset, se->discard_map))
589 return false;
590
591 err = f2fs_issue_discard(sbi, blkaddr, 1);
592 }
593
594 if (err) {
595 update_meta_page(sbi, NULL, blkaddr);
596 return true;
597 }
598 return false;
599 }
600
601 static void __add_discard_entry(struct f2fs_sb_info *sbi,
602 struct cp_control *cpc, struct seg_entry *se,
603 unsigned int start, unsigned int end)
604 {
605 struct list_head *head = &SM_I(sbi)->discard_list;
606 struct discard_entry *new, *last;
607
608 if (!list_empty(head)) {
609 last = list_last_entry(head, struct discard_entry, list);
610 if (START_BLOCK(sbi, cpc->trim_start) + start ==
611 last->blkaddr + last->len) {
612 last->len += end - start;
613 goto done;
614 }
615 }
616
617 new = f2fs_kmem_cache_alloc(discard_entry_slab, GFP_NOFS);
618 INIT_LIST_HEAD(&new->list);
619 new->blkaddr = START_BLOCK(sbi, cpc->trim_start) + start;
620 new->len = end - start;
621 list_add_tail(&new->list, head);
622 done:
623 SM_I(sbi)->nr_discards += end - start;
624 }
625
626 static void add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc)
627 {
628 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
629 int max_blocks = sbi->blocks_per_seg;
630 struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);
631 unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
632 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
633 unsigned long *discard_map = (unsigned long *)se->discard_map;
634 unsigned long *dmap = SIT_I(sbi)->tmp_map;
635 unsigned int start = 0, end = -1;
636 bool force = (cpc->reason == CP_DISCARD);
637 int i;
638
639 if (se->valid_blocks == max_blocks)
640 return;
641
642 if (!force) {
643 if (!test_opt(sbi, DISCARD) || !se->valid_blocks ||
644 SM_I(sbi)->nr_discards >= SM_I(sbi)->max_discards)
645 return;
646 }
647
648 /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */
649 for (i = 0; i < entries; i++)
650 dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] :
651 (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i];
652
653 while (force || SM_I(sbi)->nr_discards <= SM_I(sbi)->max_discards) {
654 start = __find_rev_next_bit(dmap, max_blocks, end + 1);
655 if (start >= max_blocks)
656 break;
657
658 end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1);
659 __add_discard_entry(sbi, cpc, se, start, end);
660 }
661 }
662
663 void release_discard_addrs(struct f2fs_sb_info *sbi)
664 {
665 struct list_head *head = &(SM_I(sbi)->discard_list);
666 struct discard_entry *entry, *this;
667
668 /* drop caches */
669 list_for_each_entry_safe(entry, this, head, list) {
670 list_del(&entry->list);
671 kmem_cache_free(discard_entry_slab, entry);
672 }
673 }
674
675 /*
676 * Should call clear_prefree_segments after checkpoint is done.
677 */
678 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
679 {
680 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
681 unsigned int segno;
682
683 mutex_lock(&dirty_i->seglist_lock);
684 for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))
685 __set_test_and_free(sbi, segno);
686 mutex_unlock(&dirty_i->seglist_lock);
687 }
688
689 void clear_prefree_segments(struct f2fs_sb_info *sbi, struct cp_control *cpc)
690 {
691 struct list_head *head = &(SM_I(sbi)->discard_list);
692 struct discard_entry *entry, *this;
693 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
694 unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
695 unsigned int start = 0, end = -1;
696
697 mutex_lock(&dirty_i->seglist_lock);
698
699 while (1) {
700 int i;
701 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);
702 if (start >= MAIN_SEGS(sbi))
703 break;
704 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
705 start + 1);
706
707 for (i = start; i < end; i++)
708 clear_bit(i, prefree_map);
709
710 dirty_i->nr_dirty[PRE] -= end - start;
711
712 if (!test_opt(sbi, DISCARD))
713 continue;
714
715 f2fs_issue_discard(sbi, START_BLOCK(sbi, start),
716 (end - start) << sbi->log_blocks_per_seg);
717 }
718 mutex_unlock(&dirty_i->seglist_lock);
719
720 /* send small discards */
721 list_for_each_entry_safe(entry, this, head, list) {
722 if (cpc->reason == CP_DISCARD && entry->len < cpc->trim_minlen)
723 goto skip;
724 f2fs_issue_discard(sbi, entry->blkaddr, entry->len);
725 cpc->trimmed += entry->len;
726 skip:
727 list_del(&entry->list);
728 SM_I(sbi)->nr_discards -= entry->len;
729 kmem_cache_free(discard_entry_slab, entry);
730 }
731 }
732
733 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
734 {
735 struct sit_info *sit_i = SIT_I(sbi);
736
737 if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) {
738 sit_i->dirty_sentries++;
739 return false;
740 }
741
742 return true;
743 }
744
745 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
746 unsigned int segno, int modified)
747 {
748 struct seg_entry *se = get_seg_entry(sbi, segno);
749 se->type = type;
750 if (modified)
751 __mark_sit_entry_dirty(sbi, segno);
752 }
753
754 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
755 {
756 struct seg_entry *se;
757 unsigned int segno, offset;
758 long int new_vblocks;
759
760 segno = GET_SEGNO(sbi, blkaddr);
761
762 se = get_seg_entry(sbi, segno);
763 new_vblocks = se->valid_blocks + del;
764 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
765
766 f2fs_bug_on(sbi, (new_vblocks >> (sizeof(unsigned short) << 3) ||
767 (new_vblocks > sbi->blocks_per_seg)));
768
769 se->valid_blocks = new_vblocks;
770 se->mtime = get_mtime(sbi);
771 SIT_I(sbi)->max_mtime = se->mtime;
772
773 /* Update valid block bitmap */
774 if (del > 0) {
775 if (f2fs_test_and_set_bit(offset, se->cur_valid_map))
776 f2fs_bug_on(sbi, 1);
777 if (!f2fs_test_and_set_bit(offset, se->discard_map))
778 sbi->discard_blks--;
779 } else {
780 if (!f2fs_test_and_clear_bit(offset, se->cur_valid_map))
781 f2fs_bug_on(sbi, 1);
782 if (f2fs_test_and_clear_bit(offset, se->discard_map))
783 sbi->discard_blks++;
784 }
785 if (!f2fs_test_bit(offset, se->ckpt_valid_map))
786 se->ckpt_valid_blocks += del;
787
788 __mark_sit_entry_dirty(sbi, segno);
789
790 /* update total number of valid blocks to be written in ckpt area */
791 SIT_I(sbi)->written_valid_blocks += del;
792
793 if (sbi->segs_per_sec > 1)
794 get_sec_entry(sbi, segno)->valid_blocks += del;
795 }
796
797 void refresh_sit_entry(struct f2fs_sb_info *sbi, block_t old, block_t new)
798 {
799 update_sit_entry(sbi, new, 1);
800 if (GET_SEGNO(sbi, old) != NULL_SEGNO)
801 update_sit_entry(sbi, old, -1);
802
803 locate_dirty_segment(sbi, GET_SEGNO(sbi, old));
804 locate_dirty_segment(sbi, GET_SEGNO(sbi, new));
805 }
806
807 void invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)
808 {
809 unsigned int segno = GET_SEGNO(sbi, addr);
810 struct sit_info *sit_i = SIT_I(sbi);
811
812 f2fs_bug_on(sbi, addr == NULL_ADDR);
813 if (addr == NEW_ADDR)
814 return;
815
816 /* add it into sit main buffer */
817 mutex_lock(&sit_i->sentry_lock);
818
819 update_sit_entry(sbi, addr, -1);
820
821 /* add it into dirty seglist */
822 locate_dirty_segment(sbi, segno);
823
824 mutex_unlock(&sit_i->sentry_lock);
825 }
826
827 bool is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)
828 {
829 struct sit_info *sit_i = SIT_I(sbi);
830 unsigned int segno, offset;
831 struct seg_entry *se;
832 bool is_cp = false;
833
834 if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR)
835 return true;
836
837 mutex_lock(&sit_i->sentry_lock);
838
839 segno = GET_SEGNO(sbi, blkaddr);
840 se = get_seg_entry(sbi, segno);
841 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
842
843 if (f2fs_test_bit(offset, se->ckpt_valid_map))
844 is_cp = true;
845
846 mutex_unlock(&sit_i->sentry_lock);
847
848 return is_cp;
849 }
850
851 /*
852 * This function should be resided under the curseg_mutex lock
853 */
854 static void __add_sum_entry(struct f2fs_sb_info *sbi, int type,
855 struct f2fs_summary *sum)
856 {
857 struct curseg_info *curseg = CURSEG_I(sbi, type);
858 void *addr = curseg->sum_blk;
859 addr += curseg->next_blkoff * sizeof(struct f2fs_summary);
860 memcpy(addr, sum, sizeof(struct f2fs_summary));
861 }
862
863 /*
864 * Calculate the number of current summary pages for writing
865 */
866 int npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
867 {
868 int valid_sum_count = 0;
869 int i, sum_in_page;
870
871 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
872 if (sbi->ckpt->alloc_type[i] == SSR)
873 valid_sum_count += sbi->blocks_per_seg;
874 else {
875 if (for_ra)
876 valid_sum_count += le16_to_cpu(
877 F2FS_CKPT(sbi)->cur_data_blkoff[i]);
878 else
879 valid_sum_count += curseg_blkoff(sbi, i);
880 }
881 }
882
883 sum_in_page = (PAGE_CACHE_SIZE - 2 * SUM_JOURNAL_SIZE -
884 SUM_FOOTER_SIZE) / SUMMARY_SIZE;
885 if (valid_sum_count <= sum_in_page)
886 return 1;
887 else if ((valid_sum_count - sum_in_page) <=
888 (PAGE_CACHE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE)
889 return 2;
890 return 3;
891 }
892
893 /*
894 * Caller should put this summary page
895 */
896 struct page *get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)
897 {
898 return get_meta_page(sbi, GET_SUM_BLOCK(sbi, segno));
899 }
900
901 void update_meta_page(struct f2fs_sb_info *sbi, void *src, block_t blk_addr)
902 {
903 struct page *page = grab_meta_page(sbi, blk_addr);
904 void *dst = page_address(page);
905
906 if (src)
907 memcpy(dst, src, PAGE_CACHE_SIZE);
908 else
909 memset(dst, 0, PAGE_CACHE_SIZE);
910 set_page_dirty(page);
911 f2fs_put_page(page, 1);
912 }
913
914 static void write_sum_page(struct f2fs_sb_info *sbi,
915 struct f2fs_summary_block *sum_blk, block_t blk_addr)
916 {
917 update_meta_page(sbi, (void *)sum_blk, blk_addr);
918 }
919
920 static int is_next_segment_free(struct f2fs_sb_info *sbi, int type)
921 {
922 struct curseg_info *curseg = CURSEG_I(sbi, type);
923 unsigned int segno = curseg->segno + 1;
924 struct free_segmap_info *free_i = FREE_I(sbi);
925
926 if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec)
927 return !test_bit(segno, free_i->free_segmap);
928 return 0;
929 }
930
931 /*
932 * Find a new segment from the free segments bitmap to right order
933 * This function should be returned with success, otherwise BUG
934 */
935 static void get_new_segment(struct f2fs_sb_info *sbi,
936 unsigned int *newseg, bool new_sec, int dir)
937 {
938 struct free_segmap_info *free_i = FREE_I(sbi);
939 unsigned int segno, secno, zoneno;
940 unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone;
941 unsigned int hint = *newseg / sbi->segs_per_sec;
942 unsigned int old_zoneno = GET_ZONENO_FROM_SEGNO(sbi, *newseg);
943 unsigned int left_start = hint;
944 bool init = true;
945 int go_left = 0;
946 int i;
947
948 spin_lock(&free_i->segmap_lock);
949
950 if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {
951 segno = find_next_zero_bit(free_i->free_segmap,
952 (hint + 1) * sbi->segs_per_sec, *newseg + 1);
953 if (segno < (hint + 1) * sbi->segs_per_sec)
954 goto got_it;
955 }
956 find_other_zone:
957 secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);
958 if (secno >= MAIN_SECS(sbi)) {
959 if (dir == ALLOC_RIGHT) {
960 secno = find_next_zero_bit(free_i->free_secmap,
961 MAIN_SECS(sbi), 0);
962 f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi));
963 } else {
964 go_left = 1;
965 left_start = hint - 1;
966 }
967 }
968 if (go_left == 0)
969 goto skip_left;
970
971 while (test_bit(left_start, free_i->free_secmap)) {
972 if (left_start > 0) {
973 left_start--;
974 continue;
975 }
976 left_start = find_next_zero_bit(free_i->free_secmap,
977 MAIN_SECS(sbi), 0);
978 f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi));
979 break;
980 }
981 secno = left_start;
982 skip_left:
983 hint = secno;
984 segno = secno * sbi->segs_per_sec;
985 zoneno = secno / sbi->secs_per_zone;
986
987 /* give up on finding another zone */
988 if (!init)
989 goto got_it;
990 if (sbi->secs_per_zone == 1)
991 goto got_it;
992 if (zoneno == old_zoneno)
993 goto got_it;
994 if (dir == ALLOC_LEFT) {
995 if (!go_left && zoneno + 1 >= total_zones)
996 goto got_it;
997 if (go_left && zoneno == 0)
998 goto got_it;
999 }
1000 for (i = 0; i < NR_CURSEG_TYPE; i++)
1001 if (CURSEG_I(sbi, i)->zone == zoneno)
1002 break;
1003
1004 if (i < NR_CURSEG_TYPE) {
1005 /* zone is in user, try another */
1006 if (go_left)
1007 hint = zoneno * sbi->secs_per_zone - 1;
1008 else if (zoneno + 1 >= total_zones)
1009 hint = 0;
1010 else
1011 hint = (zoneno + 1) * sbi->secs_per_zone;
1012 init = false;
1013 goto find_other_zone;
1014 }
1015 got_it:
1016 /* set it as dirty segment in free segmap */
1017 f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap));
1018 __set_inuse(sbi, segno);
1019 *newseg = segno;
1020 spin_unlock(&free_i->segmap_lock);
1021 }
1022
1023 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
1024 {
1025 struct curseg_info *curseg = CURSEG_I(sbi, type);
1026 struct summary_footer *sum_footer;
1027
1028 curseg->segno = curseg->next_segno;
1029 curseg->zone = GET_ZONENO_FROM_SEGNO(sbi, curseg->segno);
1030 curseg->next_blkoff = 0;
1031 curseg->next_segno = NULL_SEGNO;
1032
1033 sum_footer = &(curseg->sum_blk->footer);
1034 memset(sum_footer, 0, sizeof(struct summary_footer));
1035 if (IS_DATASEG(type))
1036 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
1037 if (IS_NODESEG(type))
1038 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
1039 __set_sit_entry_type(sbi, type, curseg->segno, modified);
1040 }
1041
1042 /*
1043 * Allocate a current working segment.
1044 * This function always allocates a free segment in LFS manner.
1045 */
1046 static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
1047 {
1048 struct curseg_info *curseg = CURSEG_I(sbi, type);
1049 unsigned int segno = curseg->segno;
1050 int dir = ALLOC_LEFT;
1051
1052 write_sum_page(sbi, curseg->sum_blk,
1053 GET_SUM_BLOCK(sbi, segno));
1054 if (type == CURSEG_WARM_DATA || type == CURSEG_COLD_DATA)
1055 dir = ALLOC_RIGHT;
1056
1057 if (test_opt(sbi, NOHEAP))
1058 dir = ALLOC_RIGHT;
1059
1060 get_new_segment(sbi, &segno, new_sec, dir);
1061 curseg->next_segno = segno;
1062 reset_curseg(sbi, type, 1);
1063 curseg->alloc_type = LFS;
1064 }
1065
1066 static void __next_free_blkoff(struct f2fs_sb_info *sbi,
1067 struct curseg_info *seg, block_t start)
1068 {
1069 struct seg_entry *se = get_seg_entry(sbi, seg->segno);
1070 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
1071 unsigned long *target_map = SIT_I(sbi)->tmp_map;
1072 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
1073 unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
1074 int i, pos;
1075
1076 for (i = 0; i < entries; i++)
1077 target_map[i] = ckpt_map[i] | cur_map[i];
1078
1079 pos = __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);
1080
1081 seg->next_blkoff = pos;
1082 }
1083
1084 /*
1085 * If a segment is written by LFS manner, next block offset is just obtained
1086 * by increasing the current block offset. However, if a segment is written by
1087 * SSR manner, next block offset obtained by calling __next_free_blkoff
1088 */
1089 static void __refresh_next_blkoff(struct f2fs_sb_info *sbi,
1090 struct curseg_info *seg)
1091 {
1092 if (seg->alloc_type == SSR)
1093 __next_free_blkoff(sbi, seg, seg->next_blkoff + 1);
1094 else
1095 seg->next_blkoff++;
1096 }
1097
1098 /*
1099 * This function always allocates a used segment(from dirty seglist) by SSR
1100 * manner, so it should recover the existing segment information of valid blocks
1101 */
1102 static void change_curseg(struct f2fs_sb_info *sbi, int type, bool reuse)
1103 {
1104 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1105 struct curseg_info *curseg = CURSEG_I(sbi, type);
1106 unsigned int new_segno = curseg->next_segno;
1107 struct f2fs_summary_block *sum_node;
1108 struct page *sum_page;
1109
1110 write_sum_page(sbi, curseg->sum_blk,
1111 GET_SUM_BLOCK(sbi, curseg->segno));
1112 __set_test_and_inuse(sbi, new_segno);
1113
1114 mutex_lock(&dirty_i->seglist_lock);
1115 __remove_dirty_segment(sbi, new_segno, PRE);
1116 __remove_dirty_segment(sbi, new_segno, DIRTY);
1117 mutex_unlock(&dirty_i->seglist_lock);
1118
1119 reset_curseg(sbi, type, 1);
1120 curseg->alloc_type = SSR;
1121 __next_free_blkoff(sbi, curseg, 0);
1122
1123 if (reuse) {
1124 sum_page = get_sum_page(sbi, new_segno);
1125 sum_node = (struct f2fs_summary_block *)page_address(sum_page);
1126 memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);
1127 f2fs_put_page(sum_page, 1);
1128 }
1129 }
1130
1131 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type)
1132 {
1133 struct curseg_info *curseg = CURSEG_I(sbi, type);
1134 const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops;
1135
1136 if (IS_NODESEG(type) || !has_not_enough_free_secs(sbi, 0))
1137 return v_ops->get_victim(sbi,
1138 &(curseg)->next_segno, BG_GC, type, SSR);
1139
1140 /* For data segments, let's do SSR more intensively */
1141 for (; type >= CURSEG_HOT_DATA; type--)
1142 if (v_ops->get_victim(sbi, &(curseg)->next_segno,
1143 BG_GC, type, SSR))
1144 return 1;
1145 return 0;
1146 }
1147
1148 /*
1149 * flush out current segment and replace it with new segment
1150 * This function should be returned with success, otherwise BUG
1151 */
1152 static void allocate_segment_by_default(struct f2fs_sb_info *sbi,
1153 int type, bool force)
1154 {
1155 struct curseg_info *curseg = CURSEG_I(sbi, type);
1156
1157 if (force)
1158 new_curseg(sbi, type, true);
1159 else if (type == CURSEG_WARM_NODE)
1160 new_curseg(sbi, type, false);
1161 else if (curseg->alloc_type == LFS && is_next_segment_free(sbi, type))
1162 new_curseg(sbi, type, false);
1163 else if (need_SSR(sbi) && get_ssr_segment(sbi, type))
1164 change_curseg(sbi, type, true);
1165 else
1166 new_curseg(sbi, type, false);
1167
1168 stat_inc_seg_type(sbi, curseg);
1169 }
1170
1171 static void __allocate_new_segments(struct f2fs_sb_info *sbi, int type)
1172 {
1173 struct curseg_info *curseg = CURSEG_I(sbi, type);
1174 unsigned int old_segno;
1175
1176 old_segno = curseg->segno;
1177 SIT_I(sbi)->s_ops->allocate_segment(sbi, type, true);
1178 locate_dirty_segment(sbi, old_segno);
1179 }
1180
1181 void allocate_new_segments(struct f2fs_sb_info *sbi)
1182 {
1183 int i;
1184
1185 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++)
1186 __allocate_new_segments(sbi, i);
1187 }
1188
1189 static const struct segment_allocation default_salloc_ops = {
1190 .allocate_segment = allocate_segment_by_default,
1191 };
1192
1193 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
1194 {
1195 __u64 start = F2FS_BYTES_TO_BLK(range->start);
1196 __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1;
1197 unsigned int start_segno, end_segno;
1198 struct cp_control cpc;
1199 int err = 0;
1200
1201 if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize)
1202 return -EINVAL;
1203
1204 cpc.trimmed = 0;
1205 if (end <= MAIN_BLKADDR(sbi))
1206 goto out;
1207
1208 /* start/end segment number in main_area */
1209 start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start);
1210 end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 :
1211 GET_SEGNO(sbi, end);
1212 cpc.reason = CP_DISCARD;
1213 cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen));
1214
1215 /* do checkpoint to issue discard commands safely */
1216 for (; start_segno <= end_segno; start_segno = cpc.trim_end + 1) {
1217 cpc.trim_start = start_segno;
1218
1219 if (sbi->discard_blks == 0)
1220 break;
1221 else if (sbi->discard_blks < BATCHED_TRIM_BLOCKS(sbi))
1222 cpc.trim_end = end_segno;
1223 else
1224 cpc.trim_end = min_t(unsigned int,
1225 rounddown(start_segno +
1226 BATCHED_TRIM_SEGMENTS(sbi),
1227 sbi->segs_per_sec) - 1, end_segno);
1228
1229 mutex_lock(&sbi->gc_mutex);
1230 err = write_checkpoint(sbi, &cpc);
1231 mutex_unlock(&sbi->gc_mutex);
1232 }
1233 out:
1234 range->len = F2FS_BLK_TO_BYTES(cpc.trimmed);
1235 return err;
1236 }
1237
1238 static bool __has_curseg_space(struct f2fs_sb_info *sbi, int type)
1239 {
1240 struct curseg_info *curseg = CURSEG_I(sbi, type);
1241 if (curseg->next_blkoff < sbi->blocks_per_seg)
1242 return true;
1243 return false;
1244 }
1245
1246 static int __get_segment_type_2(struct page *page, enum page_type p_type)
1247 {
1248 if (p_type == DATA)
1249 return CURSEG_HOT_DATA;
1250 else
1251 return CURSEG_HOT_NODE;
1252 }
1253
1254 static int __get_segment_type_4(struct page *page, enum page_type p_type)
1255 {
1256 if (p_type == DATA) {
1257 struct inode *inode = page->mapping->host;
1258
1259 if (S_ISDIR(inode->i_mode))
1260 return CURSEG_HOT_DATA;
1261 else
1262 return CURSEG_COLD_DATA;
1263 } else {
1264 if (IS_DNODE(page) && is_cold_node(page))
1265 return CURSEG_WARM_NODE;
1266 else
1267 return CURSEG_COLD_NODE;
1268 }
1269 }
1270
1271 static int __get_segment_type_6(struct page *page, enum page_type p_type)
1272 {
1273 if (p_type == DATA) {
1274 struct inode *inode = page->mapping->host;
1275
1276 if (S_ISDIR(inode->i_mode))
1277 return CURSEG_HOT_DATA;
1278 else if (is_cold_data(page) || file_is_cold(inode))
1279 return CURSEG_COLD_DATA;
1280 else
1281 return CURSEG_WARM_DATA;
1282 } else {
1283 if (IS_DNODE(page))
1284 return is_cold_node(page) ? CURSEG_WARM_NODE :
1285 CURSEG_HOT_NODE;
1286 else
1287 return CURSEG_COLD_NODE;
1288 }
1289 }
1290
1291 static int __get_segment_type(struct page *page, enum page_type p_type)
1292 {
1293 switch (F2FS_P_SB(page)->active_logs) {
1294 case 2:
1295 return __get_segment_type_2(page, p_type);
1296 case 4:
1297 return __get_segment_type_4(page, p_type);
1298 }
1299 /* NR_CURSEG_TYPE(6) logs by default */
1300 f2fs_bug_on(F2FS_P_SB(page),
1301 F2FS_P_SB(page)->active_logs != NR_CURSEG_TYPE);
1302 return __get_segment_type_6(page, p_type);
1303 }
1304
1305 void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
1306 block_t old_blkaddr, block_t *new_blkaddr,
1307 struct f2fs_summary *sum, int type)
1308 {
1309 struct sit_info *sit_i = SIT_I(sbi);
1310 struct curseg_info *curseg;
1311 bool direct_io = (type == CURSEG_DIRECT_IO);
1312
1313 type = direct_io ? CURSEG_WARM_DATA : type;
1314
1315 curseg = CURSEG_I(sbi, type);
1316
1317 mutex_lock(&curseg->curseg_mutex);
1318 mutex_lock(&sit_i->sentry_lock);
1319
1320 /* direct_io'ed data is aligned to the segment for better performance */
1321 if (direct_io && curseg->next_blkoff &&
1322 !has_not_enough_free_secs(sbi, 0))
1323 __allocate_new_segments(sbi, type);
1324
1325 *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
1326
1327 /*
1328 * __add_sum_entry should be resided under the curseg_mutex
1329 * because, this function updates a summary entry in the
1330 * current summary block.
1331 */
1332 __add_sum_entry(sbi, type, sum);
1333
1334 __refresh_next_blkoff(sbi, curseg);
1335
1336 stat_inc_block_count(sbi, curseg);
1337
1338 if (!__has_curseg_space(sbi, type))
1339 sit_i->s_ops->allocate_segment(sbi, type, false);
1340 /*
1341 * SIT information should be updated before segment allocation,
1342 * since SSR needs latest valid block information.
1343 */
1344 refresh_sit_entry(sbi, old_blkaddr, *new_blkaddr);
1345
1346 mutex_unlock(&sit_i->sentry_lock);
1347
1348 if (page && IS_NODESEG(type))
1349 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
1350
1351 mutex_unlock(&curseg->curseg_mutex);
1352 }
1353
1354 static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
1355 {
1356 int type = __get_segment_type(fio->page, fio->type);
1357
1358 allocate_data_block(fio->sbi, fio->page, fio->blk_addr,
1359 &fio->blk_addr, sum, type);
1360
1361 /* writeout dirty page into bdev */
1362 f2fs_submit_page_mbio(fio);
1363 }
1364
1365 void write_meta_page(struct f2fs_sb_info *sbi, struct page *page)
1366 {
1367 struct f2fs_io_info fio = {
1368 .sbi = sbi,
1369 .type = META,
1370 .rw = WRITE_SYNC | REQ_META | REQ_PRIO,
1371 .blk_addr = page->index,
1372 .page = page,
1373 .encrypted_page = NULL,
1374 };
1375
1376 if (unlikely(page->index >= MAIN_BLKADDR(sbi)))
1377 fio.rw &= ~REQ_META;
1378
1379 set_page_writeback(page);
1380 f2fs_submit_page_mbio(&fio);
1381 }
1382
1383 void write_node_page(unsigned int nid, struct f2fs_io_info *fio)
1384 {
1385 struct f2fs_summary sum;
1386
1387 set_summary(&sum, nid, 0, 0);
1388 do_write_page(&sum, fio);
1389 }
1390
1391 void write_data_page(struct dnode_of_data *dn, struct f2fs_io_info *fio)
1392 {
1393 struct f2fs_sb_info *sbi = fio->sbi;
1394 struct f2fs_summary sum;
1395 struct node_info ni;
1396
1397 f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);
1398 get_node_info(sbi, dn->nid, &ni);
1399 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
1400 do_write_page(&sum, fio);
1401 dn->data_blkaddr = fio->blk_addr;
1402 }
1403
1404 void rewrite_data_page(struct f2fs_io_info *fio)
1405 {
1406 stat_inc_inplace_blocks(fio->sbi);
1407 f2fs_submit_page_mbio(fio);
1408 }
1409
1410 static void __f2fs_replace_block(struct f2fs_sb_info *sbi,
1411 struct f2fs_summary *sum,
1412 block_t old_blkaddr, block_t new_blkaddr,
1413 bool recover_curseg, bool recover_newaddr)
1414 {
1415 struct sit_info *sit_i = SIT_I(sbi);
1416 struct curseg_info *curseg;
1417 unsigned int segno, old_cursegno;
1418 struct seg_entry *se;
1419 int type;
1420 unsigned short old_blkoff;
1421
1422 segno = GET_SEGNO(sbi, new_blkaddr);
1423 se = get_seg_entry(sbi, segno);
1424 type = se->type;
1425
1426 if (!recover_curseg) {
1427 /* for recovery flow */
1428 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {
1429 if (old_blkaddr == NULL_ADDR)
1430 type = CURSEG_COLD_DATA;
1431 else
1432 type = CURSEG_WARM_DATA;
1433 }
1434 } else {
1435 if (!IS_CURSEG(sbi, segno))
1436 type = CURSEG_WARM_DATA;
1437 }
1438
1439 curseg = CURSEG_I(sbi, type);
1440
1441 mutex_lock(&curseg->curseg_mutex);
1442 mutex_lock(&sit_i->sentry_lock);
1443
1444 old_cursegno = curseg->segno;
1445 old_blkoff = curseg->next_blkoff;
1446
1447 /* change the current segment */
1448 if (segno != curseg->segno) {
1449 curseg->next_segno = segno;
1450 change_curseg(sbi, type, true);
1451 }
1452
1453 curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
1454 __add_sum_entry(sbi, type, sum);
1455
1456 if (!recover_curseg || recover_newaddr)
1457 update_sit_entry(sbi, new_blkaddr, 1);
1458 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
1459 update_sit_entry(sbi, old_blkaddr, -1);
1460
1461 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
1462 locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr));
1463
1464 locate_dirty_segment(sbi, old_cursegno);
1465
1466 if (recover_curseg) {
1467 if (old_cursegno != curseg->segno) {
1468 curseg->next_segno = old_cursegno;
1469 change_curseg(sbi, type, true);
1470 }
1471 curseg->next_blkoff = old_blkoff;
1472 }
1473
1474 mutex_unlock(&sit_i->sentry_lock);
1475 mutex_unlock(&curseg->curseg_mutex);
1476 }
1477
1478 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
1479 block_t old_addr, block_t new_addr,
1480 unsigned char version, bool recover_curseg,
1481 bool recover_newaddr)
1482 {
1483 struct f2fs_summary sum;
1484
1485 set_summary(&sum, dn->nid, dn->ofs_in_node, version);
1486
1487 __f2fs_replace_block(sbi, &sum, old_addr, new_addr,
1488 recover_curseg, recover_newaddr);
1489
1490 dn->data_blkaddr = new_addr;
1491 set_data_blkaddr(dn);
1492 f2fs_update_extent_cache(dn);
1493 }
1494
1495 void f2fs_wait_on_page_writeback(struct page *page,
1496 enum page_type type, bool ordered)
1497 {
1498 if (PageWriteback(page)) {
1499 struct f2fs_sb_info *sbi = F2FS_P_SB(page);
1500
1501 f2fs_submit_merged_bio_cond(sbi, NULL, page, 0, type, WRITE);
1502 if (ordered)
1503 wait_on_page_writeback(page);
1504 else
1505 wait_for_stable_page(page);
1506 }
1507 }
1508
1509 void f2fs_wait_on_encrypted_page_writeback(struct f2fs_sb_info *sbi,
1510 block_t blkaddr)
1511 {
1512 struct page *cpage;
1513
1514 if (blkaddr == NEW_ADDR)
1515 return;
1516
1517 f2fs_bug_on(sbi, blkaddr == NULL_ADDR);
1518
1519 cpage = find_lock_page(META_MAPPING(sbi), blkaddr);
1520 if (cpage) {
1521 f2fs_wait_on_page_writeback(cpage, DATA, true);
1522 f2fs_put_page(cpage, 1);
1523 }
1524 }
1525
1526 static int read_compacted_summaries(struct f2fs_sb_info *sbi)
1527 {
1528 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1529 struct curseg_info *seg_i;
1530 unsigned char *kaddr;
1531 struct page *page;
1532 block_t start;
1533 int i, j, offset;
1534
1535 start = start_sum_block(sbi);
1536
1537 page = get_meta_page(sbi, start++);
1538 kaddr = (unsigned char *)page_address(page);
1539
1540 /* Step 1: restore nat cache */
1541 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
1542 memcpy(&seg_i->sum_blk->journal.n_nats, kaddr, SUM_JOURNAL_SIZE);
1543
1544 /* Step 2: restore sit cache */
1545 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
1546 memcpy(&seg_i->sum_blk->journal.n_sits, kaddr + SUM_JOURNAL_SIZE,
1547 SUM_JOURNAL_SIZE);
1548 offset = 2 * SUM_JOURNAL_SIZE;
1549
1550 /* Step 3: restore summary entries */
1551 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
1552 unsigned short blk_off;
1553 unsigned int segno;
1554
1555 seg_i = CURSEG_I(sbi, i);
1556 segno = le32_to_cpu(ckpt->cur_data_segno[i]);
1557 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);
1558 seg_i->next_segno = segno;
1559 reset_curseg(sbi, i, 0);
1560 seg_i->alloc_type = ckpt->alloc_type[i];
1561 seg_i->next_blkoff = blk_off;
1562
1563 if (seg_i->alloc_type == SSR)
1564 blk_off = sbi->blocks_per_seg;
1565
1566 for (j = 0; j < blk_off; j++) {
1567 struct f2fs_summary *s;
1568 s = (struct f2fs_summary *)(kaddr + offset);
1569 seg_i->sum_blk->entries[j] = *s;
1570 offset += SUMMARY_SIZE;
1571 if (offset + SUMMARY_SIZE <= PAGE_CACHE_SIZE -
1572 SUM_FOOTER_SIZE)
1573 continue;
1574
1575 f2fs_put_page(page, 1);
1576 page = NULL;
1577
1578 page = get_meta_page(sbi, start++);
1579 kaddr = (unsigned char *)page_address(page);
1580 offset = 0;
1581 }
1582 }
1583 f2fs_put_page(page, 1);
1584 return 0;
1585 }
1586
1587 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
1588 {
1589 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1590 struct f2fs_summary_block *sum;
1591 struct curseg_info *curseg;
1592 struct page *new;
1593 unsigned short blk_off;
1594 unsigned int segno = 0;
1595 block_t blk_addr = 0;
1596
1597 /* get segment number and block addr */
1598 if (IS_DATASEG(type)) {
1599 segno = le32_to_cpu(ckpt->cur_data_segno[type]);
1600 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
1601 CURSEG_HOT_DATA]);
1602 if (__exist_node_summaries(sbi))
1603 blk_addr = sum_blk_addr(sbi, NR_CURSEG_TYPE, type);
1604 else
1605 blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
1606 } else {
1607 segno = le32_to_cpu(ckpt->cur_node_segno[type -
1608 CURSEG_HOT_NODE]);
1609 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -
1610 CURSEG_HOT_NODE]);
1611 if (__exist_node_summaries(sbi))
1612 blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,
1613 type - CURSEG_HOT_NODE);
1614 else
1615 blk_addr = GET_SUM_BLOCK(sbi, segno);
1616 }
1617
1618 new = get_meta_page(sbi, blk_addr);
1619 sum = (struct f2fs_summary_block *)page_address(new);
1620
1621 if (IS_NODESEG(type)) {
1622 if (__exist_node_summaries(sbi)) {
1623 struct f2fs_summary *ns = &sum->entries[0];
1624 int i;
1625 for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {
1626 ns->version = 0;
1627 ns->ofs_in_node = 0;
1628 }
1629 } else {
1630 int err;
1631
1632 err = restore_node_summary(sbi, segno, sum);
1633 if (err) {
1634 f2fs_put_page(new, 1);
1635 return err;
1636 }
1637 }
1638 }
1639
1640 /* set uncompleted segment to curseg */
1641 curseg = CURSEG_I(sbi, type);
1642 mutex_lock(&curseg->curseg_mutex);
1643 memcpy(curseg->sum_blk, sum, PAGE_CACHE_SIZE);
1644 curseg->next_segno = segno;
1645 reset_curseg(sbi, type, 0);
1646 curseg->alloc_type = ckpt->alloc_type[type];
1647 curseg->next_blkoff = blk_off;
1648 mutex_unlock(&curseg->curseg_mutex);
1649 f2fs_put_page(new, 1);
1650 return 0;
1651 }
1652
1653 static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
1654 {
1655 int type = CURSEG_HOT_DATA;
1656 int err;
1657
1658 if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG)) {
1659 int npages = npages_for_summary_flush(sbi, true);
1660
1661 if (npages >= 2)
1662 ra_meta_pages(sbi, start_sum_block(sbi), npages,
1663 META_CP, true);
1664
1665 /* restore for compacted data summary */
1666 if (read_compacted_summaries(sbi))
1667 return -EINVAL;
1668 type = CURSEG_HOT_NODE;
1669 }
1670
1671 if (__exist_node_summaries(sbi))
1672 ra_meta_pages(sbi, sum_blk_addr(sbi, NR_CURSEG_TYPE, type),
1673 NR_CURSEG_TYPE - type, META_CP, true);
1674
1675 for (; type <= CURSEG_COLD_NODE; type++) {
1676 err = read_normal_summaries(sbi, type);
1677 if (err)
1678 return err;
1679 }
1680
1681 return 0;
1682 }
1683
1684 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
1685 {
1686 struct page *page;
1687 unsigned char *kaddr;
1688 struct f2fs_summary *summary;
1689 struct curseg_info *seg_i;
1690 int written_size = 0;
1691 int i, j;
1692
1693 page = grab_meta_page(sbi, blkaddr++);
1694 kaddr = (unsigned char *)page_address(page);
1695
1696 /* Step 1: write nat cache */
1697 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
1698 memcpy(kaddr, &seg_i->sum_blk->journal.n_nats, SUM_JOURNAL_SIZE);
1699 written_size += SUM_JOURNAL_SIZE;
1700
1701 /* Step 2: write sit cache */
1702 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
1703 memcpy(kaddr + written_size, &seg_i->sum_blk->journal.n_sits,
1704 SUM_JOURNAL_SIZE);
1705 written_size += SUM_JOURNAL_SIZE;
1706
1707 /* Step 3: write summary entries */
1708 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
1709 unsigned short blkoff;
1710 seg_i = CURSEG_I(sbi, i);
1711 if (sbi->ckpt->alloc_type[i] == SSR)
1712 blkoff = sbi->blocks_per_seg;
1713 else
1714 blkoff = curseg_blkoff(sbi, i);
1715
1716 for (j = 0; j < blkoff; j++) {
1717 if (!page) {
1718 page = grab_meta_page(sbi, blkaddr++);
1719 kaddr = (unsigned char *)page_address(page);
1720 written_size = 0;
1721 }
1722 summary = (struct f2fs_summary *)(kaddr + written_size);
1723 *summary = seg_i->sum_blk->entries[j];
1724 written_size += SUMMARY_SIZE;
1725
1726 if (written_size + SUMMARY_SIZE <= PAGE_CACHE_SIZE -
1727 SUM_FOOTER_SIZE)
1728 continue;
1729
1730 set_page_dirty(page);
1731 f2fs_put_page(page, 1);
1732 page = NULL;
1733 }
1734 }
1735 if (page) {
1736 set_page_dirty(page);
1737 f2fs_put_page(page, 1);
1738 }
1739 }
1740
1741 static void write_normal_summaries(struct f2fs_sb_info *sbi,
1742 block_t blkaddr, int type)
1743 {
1744 int i, end;
1745 if (IS_DATASEG(type))
1746 end = type + NR_CURSEG_DATA_TYPE;
1747 else
1748 end = type + NR_CURSEG_NODE_TYPE;
1749
1750 for (i = type; i < end; i++) {
1751 struct curseg_info *sum = CURSEG_I(sbi, i);
1752 mutex_lock(&sum->curseg_mutex);
1753 write_sum_page(sbi, sum->sum_blk, blkaddr + (i - type));
1754 mutex_unlock(&sum->curseg_mutex);
1755 }
1756 }
1757
1758 void write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
1759 {
1760 if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG))
1761 write_compacted_summaries(sbi, start_blk);
1762 else
1763 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);
1764 }
1765
1766 void write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
1767 {
1768 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);
1769 }
1770
1771 int lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
1772 unsigned int val, int alloc)
1773 {
1774 int i;
1775
1776 if (type == NAT_JOURNAL) {
1777 for (i = 0; i < nats_in_cursum(journal); i++) {
1778 if (le32_to_cpu(nid_in_journal(journal, i)) == val)
1779 return i;
1780 }
1781 if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL))
1782 return update_nats_in_cursum(journal, 1);
1783 } else if (type == SIT_JOURNAL) {
1784 for (i = 0; i < sits_in_cursum(journal); i++)
1785 if (le32_to_cpu(segno_in_journal(journal, i)) == val)
1786 return i;
1787 if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL))
1788 return update_sits_in_cursum(journal, 1);
1789 }
1790 return -1;
1791 }
1792
1793 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
1794 unsigned int segno)
1795 {
1796 return get_meta_page(sbi, current_sit_addr(sbi, segno));
1797 }
1798
1799 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
1800 unsigned int start)
1801 {
1802 struct sit_info *sit_i = SIT_I(sbi);
1803 struct page *src_page, *dst_page;
1804 pgoff_t src_off, dst_off;
1805 void *src_addr, *dst_addr;
1806
1807 src_off = current_sit_addr(sbi, start);
1808 dst_off = next_sit_addr(sbi, src_off);
1809
1810 /* get current sit block page without lock */
1811 src_page = get_meta_page(sbi, src_off);
1812 dst_page = grab_meta_page(sbi, dst_off);
1813 f2fs_bug_on(sbi, PageDirty(src_page));
1814
1815 src_addr = page_address(src_page);
1816 dst_addr = page_address(dst_page);
1817 memcpy(dst_addr, src_addr, PAGE_CACHE_SIZE);
1818
1819 set_page_dirty(dst_page);
1820 f2fs_put_page(src_page, 1);
1821
1822 set_to_next_sit(sit_i, start);
1823
1824 return dst_page;
1825 }
1826
1827 static struct sit_entry_set *grab_sit_entry_set(void)
1828 {
1829 struct sit_entry_set *ses =
1830 f2fs_kmem_cache_alloc(sit_entry_set_slab, GFP_NOFS);
1831
1832 ses->entry_cnt = 0;
1833 INIT_LIST_HEAD(&ses->set_list);
1834 return ses;
1835 }
1836
1837 static void release_sit_entry_set(struct sit_entry_set *ses)
1838 {
1839 list_del(&ses->set_list);
1840 kmem_cache_free(sit_entry_set_slab, ses);
1841 }
1842
1843 static void adjust_sit_entry_set(struct sit_entry_set *ses,
1844 struct list_head *head)
1845 {
1846 struct sit_entry_set *next = ses;
1847
1848 if (list_is_last(&ses->set_list, head))
1849 return;
1850
1851 list_for_each_entry_continue(next, head, set_list)
1852 if (ses->entry_cnt <= next->entry_cnt)
1853 break;
1854
1855 list_move_tail(&ses->set_list, &next->set_list);
1856 }
1857
1858 static void add_sit_entry(unsigned int segno, struct list_head *head)
1859 {
1860 struct sit_entry_set *ses;
1861 unsigned int start_segno = START_SEGNO(segno);
1862
1863 list_for_each_entry(ses, head, set_list) {
1864 if (ses->start_segno == start_segno) {
1865 ses->entry_cnt++;
1866 adjust_sit_entry_set(ses, head);
1867 return;
1868 }
1869 }
1870
1871 ses = grab_sit_entry_set();
1872
1873 ses->start_segno = start_segno;
1874 ses->entry_cnt++;
1875 list_add(&ses->set_list, head);
1876 }
1877
1878 static void add_sits_in_set(struct f2fs_sb_info *sbi)
1879 {
1880 struct f2fs_sm_info *sm_info = SM_I(sbi);
1881 struct list_head *set_list = &sm_info->sit_entry_set;
1882 unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap;
1883 unsigned int segno;
1884
1885 for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))
1886 add_sit_entry(segno, set_list);
1887 }
1888
1889 static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
1890 {
1891 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
1892 struct f2fs_journal *journal = &curseg->sum_blk->journal;
1893 int i;
1894
1895 for (i = 0; i < sits_in_cursum(journal); i++) {
1896 unsigned int segno;
1897 bool dirtied;
1898
1899 segno = le32_to_cpu(segno_in_journal(journal, i));
1900 dirtied = __mark_sit_entry_dirty(sbi, segno);
1901
1902 if (!dirtied)
1903 add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);
1904 }
1905 update_sits_in_cursum(journal, -i);
1906 }
1907
1908 /*
1909 * CP calls this function, which flushes SIT entries including sit_journal,
1910 * and moves prefree segs to free segs.
1911 */
1912 void flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
1913 {
1914 struct sit_info *sit_i = SIT_I(sbi);
1915 unsigned long *bitmap = sit_i->dirty_sentries_bitmap;
1916 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
1917 struct f2fs_journal *journal = &curseg->sum_blk->journal;
1918 struct sit_entry_set *ses, *tmp;
1919 struct list_head *head = &SM_I(sbi)->sit_entry_set;
1920 bool to_journal = true;
1921 struct seg_entry *se;
1922
1923 mutex_lock(&curseg->curseg_mutex);
1924 mutex_lock(&sit_i->sentry_lock);
1925
1926 if (!sit_i->dirty_sentries)
1927 goto out;
1928
1929 /*
1930 * add and account sit entries of dirty bitmap in sit entry
1931 * set temporarily
1932 */
1933 add_sits_in_set(sbi);
1934
1935 /*
1936 * if there are no enough space in journal to store dirty sit
1937 * entries, remove all entries from journal and add and account
1938 * them in sit entry set.
1939 */
1940 if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL))
1941 remove_sits_in_journal(sbi);
1942
1943 /*
1944 * there are two steps to flush sit entries:
1945 * #1, flush sit entries to journal in current cold data summary block.
1946 * #2, flush sit entries to sit page.
1947 */
1948 list_for_each_entry_safe(ses, tmp, head, set_list) {
1949 struct page *page = NULL;
1950 struct f2fs_sit_block *raw_sit = NULL;
1951 unsigned int start_segno = ses->start_segno;
1952 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK,
1953 (unsigned long)MAIN_SEGS(sbi));
1954 unsigned int segno = start_segno;
1955
1956 if (to_journal &&
1957 !__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL))
1958 to_journal = false;
1959
1960 if (!to_journal) {
1961 page = get_next_sit_page(sbi, start_segno);
1962 raw_sit = page_address(page);
1963 }
1964
1965 /* flush dirty sit entries in region of current sit set */
1966 for_each_set_bit_from(segno, bitmap, end) {
1967 int offset, sit_offset;
1968
1969 se = get_seg_entry(sbi, segno);
1970
1971 /* add discard candidates */
1972 if (cpc->reason != CP_DISCARD) {
1973 cpc->trim_start = segno;
1974 add_discard_addrs(sbi, cpc);
1975 }
1976
1977 if (to_journal) {
1978 offset = lookup_journal_in_cursum(journal,
1979 SIT_JOURNAL, segno, 1);
1980 f2fs_bug_on(sbi, offset < 0);
1981 segno_in_journal(journal, offset) =
1982 cpu_to_le32(segno);
1983 seg_info_to_raw_sit(se,
1984 &sit_in_journal(journal, offset));
1985 } else {
1986 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);
1987 seg_info_to_raw_sit(se,
1988 &raw_sit->entries[sit_offset]);
1989 }
1990
1991 __clear_bit(segno, bitmap);
1992 sit_i->dirty_sentries--;
1993 ses->entry_cnt--;
1994 }
1995
1996 if (!to_journal)
1997 f2fs_put_page(page, 1);
1998
1999 f2fs_bug_on(sbi, ses->entry_cnt);
2000 release_sit_entry_set(ses);
2001 }
2002
2003 f2fs_bug_on(sbi, !list_empty(head));
2004 f2fs_bug_on(sbi, sit_i->dirty_sentries);
2005 out:
2006 if (cpc->reason == CP_DISCARD) {
2007 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++)
2008 add_discard_addrs(sbi, cpc);
2009 }
2010 mutex_unlock(&sit_i->sentry_lock);
2011 mutex_unlock(&curseg->curseg_mutex);
2012
2013 set_prefree_as_free_segments(sbi);
2014 }
2015
2016 static int build_sit_info(struct f2fs_sb_info *sbi)
2017 {
2018 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
2019 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
2020 struct sit_info *sit_i;
2021 unsigned int sit_segs, start;
2022 char *src_bitmap, *dst_bitmap;
2023 unsigned int bitmap_size;
2024
2025 /* allocate memory for SIT information */
2026 sit_i = kzalloc(sizeof(struct sit_info), GFP_KERNEL);
2027 if (!sit_i)
2028 return -ENOMEM;
2029
2030 SM_I(sbi)->sit_info = sit_i;
2031
2032 sit_i->sentries = f2fs_kvzalloc(MAIN_SEGS(sbi) *
2033 sizeof(struct seg_entry), GFP_KERNEL);
2034 if (!sit_i->sentries)
2035 return -ENOMEM;
2036
2037 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
2038 sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(bitmap_size, GFP_KERNEL);
2039 if (!sit_i->dirty_sentries_bitmap)
2040 return -ENOMEM;
2041
2042 for (start = 0; start < MAIN_SEGS(sbi); start++) {
2043 sit_i->sentries[start].cur_valid_map
2044 = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2045 sit_i->sentries[start].ckpt_valid_map
2046 = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2047 sit_i->sentries[start].discard_map
2048 = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2049 if (!sit_i->sentries[start].cur_valid_map ||
2050 !sit_i->sentries[start].ckpt_valid_map ||
2051 !sit_i->sentries[start].discard_map)
2052 return -ENOMEM;
2053 }
2054
2055 sit_i->tmp_map = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2056 if (!sit_i->tmp_map)
2057 return -ENOMEM;
2058
2059 if (sbi->segs_per_sec > 1) {
2060 sit_i->sec_entries = f2fs_kvzalloc(MAIN_SECS(sbi) *
2061 sizeof(struct sec_entry), GFP_KERNEL);
2062 if (!sit_i->sec_entries)
2063 return -ENOMEM;
2064 }
2065
2066 /* get information related with SIT */
2067 sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;
2068
2069 /* setup SIT bitmap from ckeckpoint pack */
2070 bitmap_size = __bitmap_size(sbi, SIT_BITMAP);
2071 src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);
2072
2073 dst_bitmap = kmemdup(src_bitmap, bitmap_size, GFP_KERNEL);
2074 if (!dst_bitmap)
2075 return -ENOMEM;
2076
2077 /* init SIT information */
2078 sit_i->s_ops = &default_salloc_ops;
2079
2080 sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
2081 sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;
2082 sit_i->written_valid_blocks = le64_to_cpu(ckpt->valid_block_count);
2083 sit_i->sit_bitmap = dst_bitmap;
2084 sit_i->bitmap_size = bitmap_size;
2085 sit_i->dirty_sentries = 0;
2086 sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
2087 sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
2088 sit_i->mounted_time = CURRENT_TIME_SEC.tv_sec;
2089 mutex_init(&sit_i->sentry_lock);
2090 return 0;
2091 }
2092
2093 static int build_free_segmap(struct f2fs_sb_info *sbi)
2094 {
2095 struct free_segmap_info *free_i;
2096 unsigned int bitmap_size, sec_bitmap_size;
2097
2098 /* allocate memory for free segmap information */
2099 free_i = kzalloc(sizeof(struct free_segmap_info), GFP_KERNEL);
2100 if (!free_i)
2101 return -ENOMEM;
2102
2103 SM_I(sbi)->free_info = free_i;
2104
2105 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
2106 free_i->free_segmap = f2fs_kvmalloc(bitmap_size, GFP_KERNEL);
2107 if (!free_i->free_segmap)
2108 return -ENOMEM;
2109
2110 sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
2111 free_i->free_secmap = f2fs_kvmalloc(sec_bitmap_size, GFP_KERNEL);
2112 if (!free_i->free_secmap)
2113 return -ENOMEM;
2114
2115 /* set all segments as dirty temporarily */
2116 memset(free_i->free_segmap, 0xff, bitmap_size);
2117 memset(free_i->free_secmap, 0xff, sec_bitmap_size);
2118
2119 /* init free segmap information */
2120 free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi));
2121 free_i->free_segments = 0;
2122 free_i->free_sections = 0;
2123 spin_lock_init(&free_i->segmap_lock);
2124 return 0;
2125 }
2126
2127 static int build_curseg(struct f2fs_sb_info *sbi)
2128 {
2129 struct curseg_info *array;
2130 int i;
2131
2132 array = kcalloc(NR_CURSEG_TYPE, sizeof(*array), GFP_KERNEL);
2133 if (!array)
2134 return -ENOMEM;
2135
2136 SM_I(sbi)->curseg_array = array;
2137
2138 for (i = 0; i < NR_CURSEG_TYPE; i++) {
2139 mutex_init(&array[i].curseg_mutex);
2140 array[i].sum_blk = kzalloc(PAGE_CACHE_SIZE, GFP_KERNEL);
2141 if (!array[i].sum_blk)
2142 return -ENOMEM;
2143 array[i].segno = NULL_SEGNO;
2144 array[i].next_blkoff = 0;
2145 }
2146 return restore_curseg_summaries(sbi);
2147 }
2148
2149 static void build_sit_entries(struct f2fs_sb_info *sbi)
2150 {
2151 struct sit_info *sit_i = SIT_I(sbi);
2152 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
2153 struct f2fs_journal *journal = &curseg->sum_blk->journal;
2154 int sit_blk_cnt = SIT_BLK_CNT(sbi);
2155 unsigned int i, start, end;
2156 unsigned int readed, start_blk = 0;
2157 int nrpages = MAX_BIO_BLOCKS(sbi);
2158
2159 do {
2160 readed = ra_meta_pages(sbi, start_blk, nrpages, META_SIT, true);
2161
2162 start = start_blk * sit_i->sents_per_block;
2163 end = (start_blk + readed) * sit_i->sents_per_block;
2164
2165 for (; start < end && start < MAIN_SEGS(sbi); start++) {
2166 struct seg_entry *se = &sit_i->sentries[start];
2167 struct f2fs_sit_block *sit_blk;
2168 struct f2fs_sit_entry sit;
2169 struct page *page;
2170
2171 mutex_lock(&curseg->curseg_mutex);
2172 for (i = 0; i < sits_in_cursum(journal); i++) {
2173 if (le32_to_cpu(segno_in_journal(journal, i))
2174 == start) {
2175 sit = sit_in_journal(journal, i);
2176 mutex_unlock(&curseg->curseg_mutex);
2177 goto got_it;
2178 }
2179 }
2180 mutex_unlock(&curseg->curseg_mutex);
2181
2182 page = get_current_sit_page(sbi, start);
2183 sit_blk = (struct f2fs_sit_block *)page_address(page);
2184 sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
2185 f2fs_put_page(page, 1);
2186 got_it:
2187 check_block_count(sbi, start, &sit);
2188 seg_info_from_raw_sit(se, &sit);
2189
2190 /* build discard map only one time */
2191 memcpy(se->discard_map, se->cur_valid_map, SIT_VBLOCK_MAP_SIZE);
2192 sbi->discard_blks += sbi->blocks_per_seg - se->valid_blocks;
2193
2194 if (sbi->segs_per_sec > 1) {
2195 struct sec_entry *e = get_sec_entry(sbi, start);
2196 e->valid_blocks += se->valid_blocks;
2197 }
2198 }
2199 start_blk += readed;
2200 } while (start_blk < sit_blk_cnt);
2201 }
2202
2203 static void init_free_segmap(struct f2fs_sb_info *sbi)
2204 {
2205 unsigned int start;
2206 int type;
2207
2208 for (start = 0; start < MAIN_SEGS(sbi); start++) {
2209 struct seg_entry *sentry = get_seg_entry(sbi, start);
2210 if (!sentry->valid_blocks)
2211 __set_free(sbi, start);
2212 }
2213
2214 /* set use the current segments */
2215 for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
2216 struct curseg_info *curseg_t = CURSEG_I(sbi, type);
2217 __set_test_and_inuse(sbi, curseg_t->segno);
2218 }
2219 }
2220
2221 static void init_dirty_segmap(struct f2fs_sb_info *sbi)
2222 {
2223 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2224 struct free_segmap_info *free_i = FREE_I(sbi);
2225 unsigned int segno = 0, offset = 0;
2226 unsigned short valid_blocks;
2227
2228 while (1) {
2229 /* find dirty segment based on free segmap */
2230 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset);
2231 if (segno >= MAIN_SEGS(sbi))
2232 break;
2233 offset = segno + 1;
2234 valid_blocks = get_valid_blocks(sbi, segno, 0);
2235 if (valid_blocks == sbi->blocks_per_seg || !valid_blocks)
2236 continue;
2237 if (valid_blocks > sbi->blocks_per_seg) {
2238 f2fs_bug_on(sbi, 1);
2239 continue;
2240 }
2241 mutex_lock(&dirty_i->seglist_lock);
2242 __locate_dirty_segment(sbi, segno, DIRTY);
2243 mutex_unlock(&dirty_i->seglist_lock);
2244 }
2245 }
2246
2247 static int init_victim_secmap(struct f2fs_sb_info *sbi)
2248 {
2249 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2250 unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
2251
2252 dirty_i->victim_secmap = f2fs_kvzalloc(bitmap_size, GFP_KERNEL);
2253 if (!dirty_i->victim_secmap)
2254 return -ENOMEM;
2255 return 0;
2256 }
2257
2258 static int build_dirty_segmap(struct f2fs_sb_info *sbi)
2259 {
2260 struct dirty_seglist_info *dirty_i;
2261 unsigned int bitmap_size, i;
2262
2263 /* allocate memory for dirty segments list information */
2264 dirty_i = kzalloc(sizeof(struct dirty_seglist_info), GFP_KERNEL);
2265 if (!dirty_i)
2266 return -ENOMEM;
2267
2268 SM_I(sbi)->dirty_info = dirty_i;
2269 mutex_init(&dirty_i->seglist_lock);
2270
2271 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
2272
2273 for (i = 0; i < NR_DIRTY_TYPE; i++) {
2274 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(bitmap_size, GFP_KERNEL);
2275 if (!dirty_i->dirty_segmap[i])
2276 return -ENOMEM;
2277 }
2278
2279 init_dirty_segmap(sbi);
2280 return init_victim_secmap(sbi);
2281 }
2282
2283 /*
2284 * Update min, max modified time for cost-benefit GC algorithm
2285 */
2286 static void init_min_max_mtime(struct f2fs_sb_info *sbi)
2287 {
2288 struct sit_info *sit_i = SIT_I(sbi);
2289 unsigned int segno;
2290
2291 mutex_lock(&sit_i->sentry_lock);
2292
2293 sit_i->min_mtime = LLONG_MAX;
2294
2295 for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
2296 unsigned int i;
2297 unsigned long long mtime = 0;
2298
2299 for (i = 0; i < sbi->segs_per_sec; i++)
2300 mtime += get_seg_entry(sbi, segno + i)->mtime;
2301
2302 mtime = div_u64(mtime, sbi->segs_per_sec);
2303
2304 if (sit_i->min_mtime > mtime)
2305 sit_i->min_mtime = mtime;
2306 }
2307 sit_i->max_mtime = get_mtime(sbi);
2308 mutex_unlock(&sit_i->sentry_lock);
2309 }
2310
2311 int build_segment_manager(struct f2fs_sb_info *sbi)
2312 {
2313 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
2314 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
2315 struct f2fs_sm_info *sm_info;
2316 int err;
2317
2318 sm_info = kzalloc(sizeof(struct f2fs_sm_info), GFP_KERNEL);
2319 if (!sm_info)
2320 return -ENOMEM;
2321
2322 /* init sm info */
2323 sbi->sm_info = sm_info;
2324 sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);
2325 sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);
2326 sm_info->segment_count = le32_to_cpu(raw_super->segment_count);
2327 sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
2328 sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
2329 sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
2330 sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
2331 sm_info->rec_prefree_segments = sm_info->main_segments *
2332 DEF_RECLAIM_PREFREE_SEGMENTS / 100;
2333 sm_info->ipu_policy = 1 << F2FS_IPU_FSYNC;
2334 sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
2335 sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS;
2336
2337 INIT_LIST_HEAD(&sm_info->discard_list);
2338 sm_info->nr_discards = 0;
2339 sm_info->max_discards = 0;
2340
2341 sm_info->trim_sections = DEF_BATCHED_TRIM_SECTIONS;
2342
2343 INIT_LIST_HEAD(&sm_info->sit_entry_set);
2344
2345 if (test_opt(sbi, FLUSH_MERGE) && !f2fs_readonly(sbi->sb)) {
2346 err = create_flush_cmd_control(sbi);
2347 if (err)
2348 return err;
2349 }
2350
2351 err = build_sit_info(sbi);
2352 if (err)
2353 return err;
2354 err = build_free_segmap(sbi);
2355 if (err)
2356 return err;
2357 err = build_curseg(sbi);
2358 if (err)
2359 return err;
2360
2361 /* reinit free segmap based on SIT */
2362 build_sit_entries(sbi);
2363
2364 init_free_segmap(sbi);
2365 err = build_dirty_segmap(sbi);
2366 if (err)
2367 return err;
2368
2369 init_min_max_mtime(sbi);
2370 return 0;
2371 }
2372
2373 static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
2374 enum dirty_type dirty_type)
2375 {
2376 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2377
2378 mutex_lock(&dirty_i->seglist_lock);
2379 kvfree(dirty_i->dirty_segmap[dirty_type]);
2380 dirty_i->nr_dirty[dirty_type] = 0;
2381 mutex_unlock(&dirty_i->seglist_lock);
2382 }
2383
2384 static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
2385 {
2386 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2387 kvfree(dirty_i->victim_secmap);
2388 }
2389
2390 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
2391 {
2392 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2393 int i;
2394
2395 if (!dirty_i)
2396 return;
2397
2398 /* discard pre-free/dirty segments list */
2399 for (i = 0; i < NR_DIRTY_TYPE; i++)
2400 discard_dirty_segmap(sbi, i);
2401
2402 destroy_victim_secmap(sbi);
2403 SM_I(sbi)->dirty_info = NULL;
2404 kfree(dirty_i);
2405 }
2406
2407 static void destroy_curseg(struct f2fs_sb_info *sbi)
2408 {
2409 struct curseg_info *array = SM_I(sbi)->curseg_array;
2410 int i;
2411
2412 if (!array)
2413 return;
2414 SM_I(sbi)->curseg_array = NULL;
2415 for (i = 0; i < NR_CURSEG_TYPE; i++)
2416 kfree(array[i].sum_blk);
2417 kfree(array);
2418 }
2419
2420 static void destroy_free_segmap(struct f2fs_sb_info *sbi)
2421 {
2422 struct free_segmap_info *free_i = SM_I(sbi)->free_info;
2423 if (!free_i)
2424 return;
2425 SM_I(sbi)->free_info = NULL;
2426 kvfree(free_i->free_segmap);
2427 kvfree(free_i->free_secmap);
2428 kfree(free_i);
2429 }
2430
2431 static void destroy_sit_info(struct f2fs_sb_info *sbi)
2432 {
2433 struct sit_info *sit_i = SIT_I(sbi);
2434 unsigned int start;
2435
2436 if (!sit_i)
2437 return;
2438
2439 if (sit_i->sentries) {
2440 for (start = 0; start < MAIN_SEGS(sbi); start++) {
2441 kfree(sit_i->sentries[start].cur_valid_map);
2442 kfree(sit_i->sentries[start].ckpt_valid_map);
2443 kfree(sit_i->sentries[start].discard_map);
2444 }
2445 }
2446 kfree(sit_i->tmp_map);
2447
2448 kvfree(sit_i->sentries);
2449 kvfree(sit_i->sec_entries);
2450 kvfree(sit_i->dirty_sentries_bitmap);
2451
2452 SM_I(sbi)->sit_info = NULL;
2453 kfree(sit_i->sit_bitmap);
2454 kfree(sit_i);
2455 }
2456
2457 void destroy_segment_manager(struct f2fs_sb_info *sbi)
2458 {
2459 struct f2fs_sm_info *sm_info = SM_I(sbi);
2460
2461 if (!sm_info)
2462 return;
2463 destroy_flush_cmd_control(sbi);
2464 destroy_dirty_segmap(sbi);
2465 destroy_curseg(sbi);
2466 destroy_free_segmap(sbi);
2467 destroy_sit_info(sbi);
2468 sbi->sm_info = NULL;
2469 kfree(sm_info);
2470 }
2471
2472 int __init create_segment_manager_caches(void)
2473 {
2474 discard_entry_slab = f2fs_kmem_cache_create("discard_entry",
2475 sizeof(struct discard_entry));
2476 if (!discard_entry_slab)
2477 goto fail;
2478
2479 sit_entry_set_slab = f2fs_kmem_cache_create("sit_entry_set",
2480 sizeof(struct sit_entry_set));
2481 if (!sit_entry_set_slab)
2482 goto destory_discard_entry;
2483
2484 inmem_entry_slab = f2fs_kmem_cache_create("inmem_page_entry",
2485 sizeof(struct inmem_pages));
2486 if (!inmem_entry_slab)
2487 goto destroy_sit_entry_set;
2488 return 0;
2489
2490 destroy_sit_entry_set:
2491 kmem_cache_destroy(sit_entry_set_slab);
2492 destory_discard_entry:
2493 kmem_cache_destroy(discard_entry_slab);
2494 fail:
2495 return -ENOMEM;
2496 }
2497
2498 void destroy_segment_manager_caches(void)
2499 {
2500 kmem_cache_destroy(sit_entry_set_slab);
2501 kmem_cache_destroy(discard_entry_slab);
2502 kmem_cache_destroy(inmem_entry_slab);
2503 }
Linux kernel - Deliverables
This page took 0.153356 seconds and 5 git commands to generate.