projects / deliverable / linux.git / blob
? search:


770cdc95120f377543483b89e49364abd807fc78
[deliverable/linux.git] / fs / f2fs / segment.c
1 /*
2 * fs/f2fs/segment.c
3 *
4 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
5 * http://www.samsung.com/
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 */
11 #include <linux/fs.h>
12 #include <linux/f2fs_fs.h>
13 #include <linux/bio.h>
14 #include <linux/blkdev.h>
15 #include <linux/prefetch.h>
16 #include <linux/kthread.h>
17 #include <linux/swap.h>
18 #include <linux/timer.h>
19
20 #include "f2fs.h"
21 #include "segment.h"
22 #include "node.h"
23 #include "trace.h"
24 #include <trace/events/f2fs.h>
25
26 #define __reverse_ffz(x) __reverse_ffs(~(x))
27
28 static struct kmem_cache *discard_entry_slab;
29 static struct kmem_cache *sit_entry_set_slab;
30 static struct kmem_cache *inmem_entry_slab;
31
32 static unsigned long __reverse_ulong(unsigned char *str)
33 {
34 unsigned long tmp = 0;
35 int shift = 24, idx = 0;
36
37 #if BITS_PER_LONG == 64
38 shift = 56;
39 #endif
40 while (shift >= 0) {
41 tmp |= (unsigned long)str[idx++] << shift;
42 shift -= BITS_PER_BYTE;
43 }
44 return tmp;
45 }
46
47 /*
48 * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since
49 * MSB and LSB are reversed in a byte by f2fs_set_bit.
50 */
51 static inline unsigned long __reverse_ffs(unsigned long word)
52 {
53 int num = 0;
54
55 #if BITS_PER_LONG == 64
56 if ((word & 0xffffffff00000000UL) == 0)
57 num += 32;
58 else
59 word >>= 32;
60 #endif
61 if ((word & 0xffff0000) == 0)
62 num += 16;
63 else
64 word >>= 16;
65
66 if ((word & 0xff00) == 0)
67 num += 8;
68 else
69 word >>= 8;
70
71 if ((word & 0xf0) == 0)
72 num += 4;
73 else
74 word >>= 4;
75
76 if ((word & 0xc) == 0)
77 num += 2;
78 else
79 word >>= 2;
80
81 if ((word & 0x2) == 0)
82 num += 1;
83 return num;
84 }
85
86 /*
87 * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because
88 * f2fs_set_bit makes MSB and LSB reversed in a byte.
89 * @size must be integral times of unsigned long.
90 * Example:
91 * MSB <--> LSB
92 * f2fs_set_bit(0, bitmap) => 1000 0000
93 * f2fs_set_bit(7, bitmap) => 0000 0001
94 */
95 static unsigned long __find_rev_next_bit(const unsigned long *addr,
96 unsigned long size, unsigned long offset)
97 {
98 const unsigned long *p = addr + BIT_WORD(offset);
99 unsigned long result = size;
100 unsigned long tmp;
101
102 if (offset >= size)
103 return size;
104
105 size -= (offset & ~(BITS_PER_LONG - 1));
106 offset %= BITS_PER_LONG;
107
108 while (1) {
109 if (*p == 0)
110 goto pass;
111
112 tmp = __reverse_ulong((unsigned char *)p);
113
114 tmp &= ~0UL >> offset;
115 if (size < BITS_PER_LONG)
116 tmp &= (~0UL << (BITS_PER_LONG - size));
117 if (tmp)
118 goto found;
119 pass:
120 if (size <= BITS_PER_LONG)
121 break;
122 size -= BITS_PER_LONG;
123 offset = 0;
124 p++;
125 }
126 return result;
127 found:
128 return result - size + __reverse_ffs(tmp);
129 }
130
131 static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
132 unsigned long size, unsigned long offset)
133 {
134 const unsigned long *p = addr + BIT_WORD(offset);
135 unsigned long result = size;
136 unsigned long tmp;
137
138 if (offset >= size)
139 return size;
140
141 size -= (offset & ~(BITS_PER_LONG - 1));
142 offset %= BITS_PER_LONG;
143
144 while (1) {
145 if (*p == ~0UL)
146 goto pass;
147
148 tmp = __reverse_ulong((unsigned char *)p);
149
150 if (offset)
151 tmp |= ~0UL << (BITS_PER_LONG - offset);
152 if (size < BITS_PER_LONG)
153 tmp |= ~0UL >> size;
154 if (tmp != ~0UL)
155 goto found;
156 pass:
157 if (size <= BITS_PER_LONG)
158 break;
159 size -= BITS_PER_LONG;
160 offset = 0;
161 p++;
162 }
163 return result;
164 found:
165 return result - size + __reverse_ffz(tmp);
166 }
167
168 void register_inmem_page(struct inode *inode, struct page *page)
169 {
170 struct f2fs_inode_info *fi = F2FS_I(inode);
171 struct inmem_pages *new;
172
173 f2fs_trace_pid(page);
174
175 set_page_private(page, (unsigned long)ATOMIC_WRITTEN_PAGE);
176 SetPagePrivate(page);
177
178 new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS);
179
180 /* add atomic page indices to the list */
181 new->page = page;
182 INIT_LIST_HEAD(&new->list);
183
184 /* increase reference count with clean state */
185 mutex_lock(&fi->inmem_lock);
186 get_page(page);
187 list_add_tail(&new->list, &fi->inmem_pages);
188 inc_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
189 mutex_unlock(&fi->inmem_lock);
190
191 trace_f2fs_register_inmem_page(page, INMEM);
192 }
193
194 static int __revoke_inmem_pages(struct inode *inode,
195 struct list_head *head, bool drop, bool recover)
196 {
197 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
198 struct inmem_pages *cur, *tmp;
199 int err = 0;
200
201 list_for_each_entry_safe(cur, tmp, head, list) {
202 struct page *page = cur->page;
203
204 if (drop)
205 trace_f2fs_commit_inmem_page(page, INMEM_DROP);
206
207 lock_page(page);
208
209 if (recover) {
210 struct dnode_of_data dn;
211 struct node_info ni;
212
213 trace_f2fs_commit_inmem_page(page, INMEM_REVOKE);
214
215 set_new_dnode(&dn, inode, NULL, NULL, 0);
216 if (get_dnode_of_data(&dn, page->index, LOOKUP_NODE)) {
217 err = -EAGAIN;
218 goto next;
219 }
220 get_node_info(sbi, dn.nid, &ni);
221 f2fs_replace_block(sbi, &dn, dn.data_blkaddr,
222 cur->old_addr, ni.version, true, true);
223 f2fs_put_dnode(&dn);
224 }
225 next:
226 /* we don't need to invalidate this in the sccessful status */
227 if (drop || recover)
228 ClearPageUptodate(page);
229 set_page_private(page, 0);
230 f2fs_put_page(page, 1);
231
232 list_del(&cur->list);
233 kmem_cache_free(inmem_entry_slab, cur);
234 dec_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
235 }
236 return err;
237 }
238
239 void drop_inmem_pages(struct inode *inode)
240 {
241 struct f2fs_inode_info *fi = F2FS_I(inode);
242
243 clear_inode_flag(F2FS_I(inode), FI_ATOMIC_FILE);
244
245 mutex_lock(&fi->inmem_lock);
246 __revoke_inmem_pages(inode, &fi->inmem_pages, true, false);
247 mutex_unlock(&fi->inmem_lock);
248 }
249
250 static int __commit_inmem_pages(struct inode *inode,
251 struct list_head *revoke_list)
252 {
253 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
254 struct f2fs_inode_info *fi = F2FS_I(inode);
255 struct inmem_pages *cur, *tmp;
256 struct f2fs_io_info fio = {
257 .sbi = sbi,
258 .type = DATA,
259 .rw = WRITE_SYNC | REQ_PRIO,
260 .encrypted_page = NULL,
261 };
262 bool submit_bio = false;
263 int err = 0;
264
265 list_for_each_entry_safe(cur, tmp, &fi->inmem_pages, list) {
266 struct page *page = cur->page;
267
268 lock_page(page);
269 if (page->mapping == inode->i_mapping) {
270 trace_f2fs_commit_inmem_page(page, INMEM);
271
272 set_page_dirty(page);
273 f2fs_wait_on_page_writeback(page, DATA, true);
274 if (clear_page_dirty_for_io(page))
275 inode_dec_dirty_pages(inode);
276
277 fio.page = page;
278 err = do_write_data_page(&fio);
279 if (err) {
280 unlock_page(page);
281 break;
282 }
283
284 /* record old blkaddr for revoking */
285 cur->old_addr = fio.old_blkaddr;
286
287 clear_cold_data(page);
288 submit_bio = true;
289 }
290 unlock_page(page);
291 list_move_tail(&cur->list, revoke_list);
292 }
293
294 if (submit_bio)
295 f2fs_submit_merged_bio_cond(sbi, inode, NULL, 0, DATA, WRITE);
296
297 if (!err)
298 __revoke_inmem_pages(inode, revoke_list, false, false);
299
300 return err;
301 }
302
303 int commit_inmem_pages(struct inode *inode)
304 {
305 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
306 struct f2fs_inode_info *fi = F2FS_I(inode);
307 struct list_head revoke_list;
308 int err;
309
310 INIT_LIST_HEAD(&revoke_list);
311 f2fs_balance_fs(sbi, true);
312 f2fs_lock_op(sbi);
313
314 mutex_lock(&fi->inmem_lock);
315 err = __commit_inmem_pages(inode, &revoke_list);
316 if (err) {
317 int ret;
318 /*
319 * try to revoke all committed pages, but still we could fail
320 * due to no memory or other reason, if that happened, EAGAIN
321 * will be returned, which means in such case, transaction is
322 * already not integrity, caller should use journal to do the
323 * recovery or rewrite & commit last transaction. For other
324 * error number, revoking was done by filesystem itself.
325 */
326 ret = __revoke_inmem_pages(inode, &revoke_list, false, true);
327 if (ret)
328 err = ret;
329
330 /* drop all uncommitted pages */
331 __revoke_inmem_pages(inode, &fi->inmem_pages, true, false);
332 }
333 mutex_unlock(&fi->inmem_lock);
334
335 f2fs_unlock_op(sbi);
336 return err;
337 }
338
339 /*
340 * This function balances dirty node and dentry pages.
341 * In addition, it controls garbage collection.
342 */
343 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
344 {
345 if (!need)
346 return;
347 /*
348 * We should do GC or end up with checkpoint, if there are so many dirty
349 * dir/node pages without enough free segments.
350 */
351 if (has_not_enough_free_secs(sbi, 0)) {
352 mutex_lock(&sbi->gc_mutex);
353 f2fs_gc(sbi, false);
354 }
355 }
356
357 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi)
358 {
359 /* try to shrink extent cache when there is no enough memory */
360 if (!available_free_memory(sbi, EXTENT_CACHE))
361 f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER);
362
363 /* check the # of cached NAT entries */
364 if (!available_free_memory(sbi, NAT_ENTRIES))
365 try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);
366
367 if (!available_free_memory(sbi, FREE_NIDS))
368 try_to_free_nids(sbi, NAT_ENTRY_PER_BLOCK * FREE_NID_PAGES);
369
370 /* checkpoint is the only way to shrink partial cached entries */
371 if (!available_free_memory(sbi, NAT_ENTRIES) ||
372 !available_free_memory(sbi, INO_ENTRIES) ||
373 excess_prefree_segs(sbi) ||
374 excess_dirty_nats(sbi) ||
375 (is_idle(sbi) && f2fs_time_over(sbi, CP_TIME))) {
376 if (test_opt(sbi, DATA_FLUSH)) {
377 struct blk_plug plug;
378
379 blk_start_plug(&plug);
380 sync_dirty_inodes(sbi, FILE_INODE);
381 blk_finish_plug(&plug);
382 }
383 f2fs_sync_fs(sbi->sb, true);
384 stat_inc_bg_cp_count(sbi->stat_info);
385 }
386 }
387
388 static int issue_flush_thread(void *data)
389 {
390 struct f2fs_sb_info *sbi = data;
391 struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
392 wait_queue_head_t *q = &fcc->flush_wait_queue;
393 repeat:
394 if (kthread_should_stop())
395 return 0;
396
397 if (!llist_empty(&fcc->issue_list)) {
398 struct bio *bio;
399 struct flush_cmd *cmd, *next;
400 int ret;
401
402 bio = f2fs_bio_alloc(0);
403
404 fcc->dispatch_list = llist_del_all(&fcc->issue_list);
405 fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list);
406
407 bio->bi_bdev = sbi->sb->s_bdev;
408 ret = submit_bio_wait(WRITE_FLUSH, bio);
409
410 llist_for_each_entry_safe(cmd, next,
411 fcc->dispatch_list, llnode) {
412 cmd->ret = ret;
413 complete(&cmd->wait);
414 }
415 bio_put(bio);
416 fcc->dispatch_list = NULL;
417 }
418
419 wait_event_interruptible(*q,
420 kthread_should_stop() || !llist_empty(&fcc->issue_list));
421 goto repeat;
422 }
423
424 int f2fs_issue_flush(struct f2fs_sb_info *sbi)
425 {
426 struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
427 struct flush_cmd cmd;
428
429 trace_f2fs_issue_flush(sbi->sb, test_opt(sbi, NOBARRIER),
430 test_opt(sbi, FLUSH_MERGE));
431
432 if (test_opt(sbi, NOBARRIER))
433 return 0;
434
435 if (!test_opt(sbi, FLUSH_MERGE)) {
436 struct bio *bio = f2fs_bio_alloc(0);
437 int ret;
438
439 bio->bi_bdev = sbi->sb->s_bdev;
440 ret = submit_bio_wait(WRITE_FLUSH, bio);
441 bio_put(bio);
442 return ret;
443 }
444
445 init_completion(&cmd.wait);
446
447 llist_add(&cmd.llnode, &fcc->issue_list);
448
449 if (!fcc->dispatch_list)
450 wake_up(&fcc->flush_wait_queue);
451
452 wait_for_completion(&cmd.wait);
453
454 return cmd.ret;
455 }
456
457 int create_flush_cmd_control(struct f2fs_sb_info *sbi)
458 {
459 dev_t dev = sbi->sb->s_bdev->bd_dev;
460 struct flush_cmd_control *fcc;
461 int err = 0;
462
463 fcc = kzalloc(sizeof(struct flush_cmd_control), GFP_KERNEL);
464 if (!fcc)
465 return -ENOMEM;
466 init_waitqueue_head(&fcc->flush_wait_queue);
467 init_llist_head(&fcc->issue_list);
468 SM_I(sbi)->cmd_control_info = fcc;
469 fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,
470 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
471 if (IS_ERR(fcc->f2fs_issue_flush)) {
472 err = PTR_ERR(fcc->f2fs_issue_flush);
473 kfree(fcc);
474 SM_I(sbi)->cmd_control_info = NULL;
475 return err;
476 }
477
478 return err;
479 }
480
481 void destroy_flush_cmd_control(struct f2fs_sb_info *sbi)
482 {
483 struct flush_cmd_control *fcc = SM_I(sbi)->cmd_control_info;
484
485 if (fcc && fcc->f2fs_issue_flush)
486 kthread_stop(fcc->f2fs_issue_flush);
487 kfree(fcc);
488 SM_I(sbi)->cmd_control_info = NULL;
489 }
490
491 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
492 enum dirty_type dirty_type)
493 {
494 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
495
496 /* need not be added */
497 if (IS_CURSEG(sbi, segno))
498 return;
499
500 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
501 dirty_i->nr_dirty[dirty_type]++;
502
503 if (dirty_type == DIRTY) {
504 struct seg_entry *sentry = get_seg_entry(sbi, segno);
505 enum dirty_type t = sentry->type;
506
507 if (unlikely(t >= DIRTY)) {
508 f2fs_bug_on(sbi, 1);
509 return;
510 }
511 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t]))
512 dirty_i->nr_dirty[t]++;
513 }
514 }
515
516 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
517 enum dirty_type dirty_type)
518 {
519 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
520
521 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
522 dirty_i->nr_dirty[dirty_type]--;
523
524 if (dirty_type == DIRTY) {
525 struct seg_entry *sentry = get_seg_entry(sbi, segno);
526 enum dirty_type t = sentry->type;
527
528 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
529 dirty_i->nr_dirty[t]--;
530
531 if (get_valid_blocks(sbi, segno, sbi->segs_per_sec) == 0)
532 clear_bit(GET_SECNO(sbi, segno),
533 dirty_i->victim_secmap);
534 }
535 }
536
537 /*
538 * Should not occur error such as -ENOMEM.
539 * Adding dirty entry into seglist is not critical operation.
540 * If a given segment is one of current working segments, it won't be added.
541 */
542 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
543 {
544 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
545 unsigned short valid_blocks;
546
547 if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))
548 return;
549
550 mutex_lock(&dirty_i->seglist_lock);
551
552 valid_blocks = get_valid_blocks(sbi, segno, 0);
553
554 if (valid_blocks == 0) {
555 __locate_dirty_segment(sbi, segno, PRE);
556 __remove_dirty_segment(sbi, segno, DIRTY);
557 } else if (valid_blocks < sbi->blocks_per_seg) {
558 __locate_dirty_segment(sbi, segno, DIRTY);
559 } else {
560 /* Recovery routine with SSR needs this */
561 __remove_dirty_segment(sbi, segno, DIRTY);
562 }
563
564 mutex_unlock(&dirty_i->seglist_lock);
565 }
566
567 static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
568 block_t blkstart, block_t blklen)
569 {
570 sector_t start = SECTOR_FROM_BLOCK(blkstart);
571 sector_t len = SECTOR_FROM_BLOCK(blklen);
572 struct seg_entry *se;
573 unsigned int offset;
574 block_t i;
575
576 for (i = blkstart; i < blkstart + blklen; i++) {
577 se = get_seg_entry(sbi, GET_SEGNO(sbi, i));
578 offset = GET_BLKOFF_FROM_SEG0(sbi, i);
579
580 if (!f2fs_test_and_set_bit(offset, se->discard_map))
581 sbi->discard_blks--;
582 }
583 trace_f2fs_issue_discard(sbi->sb, blkstart, blklen);
584 return blkdev_issue_discard(sbi->sb->s_bdev, start, len, GFP_NOFS, 0);
585 }
586
587 bool discard_next_dnode(struct f2fs_sb_info *sbi, block_t blkaddr)
588 {
589 int err = -EOPNOTSUPP;
590
591 if (test_opt(sbi, DISCARD)) {
592 struct seg_entry *se = get_seg_entry(sbi,
593 GET_SEGNO(sbi, blkaddr));
594 unsigned int offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
595
596 if (f2fs_test_bit(offset, se->discard_map))
597 return false;
598
599 err = f2fs_issue_discard(sbi, blkaddr, 1);
600 }
601
602 if (err) {
603 update_meta_page(sbi, NULL, blkaddr);
604 return true;
605 }
606 return false;
607 }
608
609 static void __add_discard_entry(struct f2fs_sb_info *sbi,
610 struct cp_control *cpc, struct seg_entry *se,
611 unsigned int start, unsigned int end)
612 {
613 struct list_head *head = &SM_I(sbi)->discard_list;
614 struct discard_entry *new, *last;
615
616 if (!list_empty(head)) {
617 last = list_last_entry(head, struct discard_entry, list);
618 if (START_BLOCK(sbi, cpc->trim_start) + start ==
619 last->blkaddr + last->len) {
620 last->len += end - start;
621 goto done;
622 }
623 }
624
625 new = f2fs_kmem_cache_alloc(discard_entry_slab, GFP_NOFS);
626 INIT_LIST_HEAD(&new->list);
627 new->blkaddr = START_BLOCK(sbi, cpc->trim_start) + start;
628 new->len = end - start;
629 list_add_tail(&new->list, head);
630 done:
631 SM_I(sbi)->nr_discards += end - start;
632 }
633
634 static void add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc)
635 {
636 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
637 int max_blocks = sbi->blocks_per_seg;
638 struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);
639 unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
640 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
641 unsigned long *discard_map = (unsigned long *)se->discard_map;
642 unsigned long *dmap = SIT_I(sbi)->tmp_map;
643 unsigned int start = 0, end = -1;
644 bool force = (cpc->reason == CP_DISCARD);
645 int i;
646
647 if (se->valid_blocks == max_blocks)
648 return;
649
650 if (!force) {
651 if (!test_opt(sbi, DISCARD) || !se->valid_blocks ||
652 SM_I(sbi)->nr_discards >= SM_I(sbi)->max_discards)
653 return;
654 }
655
656 /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */
657 for (i = 0; i < entries; i++)
658 dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] :
659 (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i];
660
661 while (force || SM_I(sbi)->nr_discards <= SM_I(sbi)->max_discards) {
662 start = __find_rev_next_bit(dmap, max_blocks, end + 1);
663 if (start >= max_blocks)
664 break;
665
666 end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1);
667 __add_discard_entry(sbi, cpc, se, start, end);
668 }
669 }
670
671 void release_discard_addrs(struct f2fs_sb_info *sbi)
672 {
673 struct list_head *head = &(SM_I(sbi)->discard_list);
674 struct discard_entry *entry, *this;
675
676 /* drop caches */
677 list_for_each_entry_safe(entry, this, head, list) {
678 list_del(&entry->list);
679 kmem_cache_free(discard_entry_slab, entry);
680 }
681 }
682
683 /*
684 * Should call clear_prefree_segments after checkpoint is done.
685 */
686 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
687 {
688 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
689 unsigned int segno;
690
691 mutex_lock(&dirty_i->seglist_lock);
692 for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))
693 __set_test_and_free(sbi, segno);
694 mutex_unlock(&dirty_i->seglist_lock);
695 }
696
697 void clear_prefree_segments(struct f2fs_sb_info *sbi, struct cp_control *cpc)
698 {
699 struct list_head *head = &(SM_I(sbi)->discard_list);
700 struct discard_entry *entry, *this;
701 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
702 unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
703 unsigned int start = 0, end = -1;
704
705 mutex_lock(&dirty_i->seglist_lock);
706
707 while (1) {
708 int i;
709 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);
710 if (start >= MAIN_SEGS(sbi))
711 break;
712 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
713 start + 1);
714
715 for (i = start; i < end; i++)
716 clear_bit(i, prefree_map);
717
718 dirty_i->nr_dirty[PRE] -= end - start;
719
720 if (!test_opt(sbi, DISCARD))
721 continue;
722
723 f2fs_issue_discard(sbi, START_BLOCK(sbi, start),
724 (end - start) << sbi->log_blocks_per_seg);
725 }
726 mutex_unlock(&dirty_i->seglist_lock);
727
728 /* send small discards */
729 list_for_each_entry_safe(entry, this, head, list) {
730 if (cpc->reason == CP_DISCARD && entry->len < cpc->trim_minlen)
731 goto skip;
732 f2fs_issue_discard(sbi, entry->blkaddr, entry->len);
733 cpc->trimmed += entry->len;
734 skip:
735 list_del(&entry->list);
736 SM_I(sbi)->nr_discards -= entry->len;
737 kmem_cache_free(discard_entry_slab, entry);
738 }
739 }
740
741 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
742 {
743 struct sit_info *sit_i = SIT_I(sbi);
744
745 if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) {
746 sit_i->dirty_sentries++;
747 return false;
748 }
749
750 return true;
751 }
752
753 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
754 unsigned int segno, int modified)
755 {
756 struct seg_entry *se = get_seg_entry(sbi, segno);
757 se->type = type;
758 if (modified)
759 __mark_sit_entry_dirty(sbi, segno);
760 }
761
762 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
763 {
764 struct seg_entry *se;
765 unsigned int segno, offset;
766 long int new_vblocks;
767
768 segno = GET_SEGNO(sbi, blkaddr);
769
770 se = get_seg_entry(sbi, segno);
771 new_vblocks = se->valid_blocks + del;
772 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
773
774 f2fs_bug_on(sbi, (new_vblocks >> (sizeof(unsigned short) << 3) ||
775 (new_vblocks > sbi->blocks_per_seg)));
776
777 se->valid_blocks = new_vblocks;
778 se->mtime = get_mtime(sbi);
779 SIT_I(sbi)->max_mtime = se->mtime;
780
781 /* Update valid block bitmap */
782 if (del > 0) {
783 if (f2fs_test_and_set_bit(offset, se->cur_valid_map))
784 f2fs_bug_on(sbi, 1);
785 if (!f2fs_test_and_set_bit(offset, se->discard_map))
786 sbi->discard_blks--;
787 } else {
788 if (!f2fs_test_and_clear_bit(offset, se->cur_valid_map))
789 f2fs_bug_on(sbi, 1);
790 if (f2fs_test_and_clear_bit(offset, se->discard_map))
791 sbi->discard_blks++;
792 }
793 if (!f2fs_test_bit(offset, se->ckpt_valid_map))
794 se->ckpt_valid_blocks += del;
795
796 __mark_sit_entry_dirty(sbi, segno);
797
798 /* update total number of valid blocks to be written in ckpt area */
799 SIT_I(sbi)->written_valid_blocks += del;
800
801 if (sbi->segs_per_sec > 1)
802 get_sec_entry(sbi, segno)->valid_blocks += del;
803 }
804
805 void refresh_sit_entry(struct f2fs_sb_info *sbi, block_t old, block_t new)
806 {
807 update_sit_entry(sbi, new, 1);
808 if (GET_SEGNO(sbi, old) != NULL_SEGNO)
809 update_sit_entry(sbi, old, -1);
810
811 locate_dirty_segment(sbi, GET_SEGNO(sbi, old));
812 locate_dirty_segment(sbi, GET_SEGNO(sbi, new));
813 }
814
815 void invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)
816 {
817 unsigned int segno = GET_SEGNO(sbi, addr);
818 struct sit_info *sit_i = SIT_I(sbi);
819
820 f2fs_bug_on(sbi, addr == NULL_ADDR);
821 if (addr == NEW_ADDR)
822 return;
823
824 /* add it into sit main buffer */
825 mutex_lock(&sit_i->sentry_lock);
826
827 update_sit_entry(sbi, addr, -1);
828
829 /* add it into dirty seglist */
830 locate_dirty_segment(sbi, segno);
831
832 mutex_unlock(&sit_i->sentry_lock);
833 }
834
835 bool is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)
836 {
837 struct sit_info *sit_i = SIT_I(sbi);
838 unsigned int segno, offset;
839 struct seg_entry *se;
840 bool is_cp = false;
841
842 if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR)
843 return true;
844
845 mutex_lock(&sit_i->sentry_lock);
846
847 segno = GET_SEGNO(sbi, blkaddr);
848 se = get_seg_entry(sbi, segno);
849 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
850
851 if (f2fs_test_bit(offset, se->ckpt_valid_map))
852 is_cp = true;
853
854 mutex_unlock(&sit_i->sentry_lock);
855
856 return is_cp;
857 }
858
859 /*
860 * This function should be resided under the curseg_mutex lock
861 */
862 static void __add_sum_entry(struct f2fs_sb_info *sbi, int type,
863 struct f2fs_summary *sum)
864 {
865 struct curseg_info *curseg = CURSEG_I(sbi, type);
866 void *addr = curseg->sum_blk;
867 addr += curseg->next_blkoff * sizeof(struct f2fs_summary);
868 memcpy(addr, sum, sizeof(struct f2fs_summary));
869 }
870
871 /*
872 * Calculate the number of current summary pages for writing
873 */
874 int npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
875 {
876 int valid_sum_count = 0;
877 int i, sum_in_page;
878
879 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
880 if (sbi->ckpt->alloc_type[i] == SSR)
881 valid_sum_count += sbi->blocks_per_seg;
882 else {
883 if (for_ra)
884 valid_sum_count += le16_to_cpu(
885 F2FS_CKPT(sbi)->cur_data_blkoff[i]);
886 else
887 valid_sum_count += curseg_blkoff(sbi, i);
888 }
889 }
890
891 sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE -
892 SUM_FOOTER_SIZE) / SUMMARY_SIZE;
893 if (valid_sum_count <= sum_in_page)
894 return 1;
895 else if ((valid_sum_count - sum_in_page) <=
896 (PAGE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE)
897 return 2;
898 return 3;
899 }
900
901 /*
902 * Caller should put this summary page
903 */
904 struct page *get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)
905 {
906 return get_meta_page(sbi, GET_SUM_BLOCK(sbi, segno));
907 }
908
909 void update_meta_page(struct f2fs_sb_info *sbi, void *src, block_t blk_addr)
910 {
911 struct page *page = grab_meta_page(sbi, blk_addr);
912 void *dst = page_address(page);
913
914 if (src)
915 memcpy(dst, src, PAGE_SIZE);
916 else
917 memset(dst, 0, PAGE_SIZE);
918 set_page_dirty(page);
919 f2fs_put_page(page, 1);
920 }
921
922 static void write_sum_page(struct f2fs_sb_info *sbi,
923 struct f2fs_summary_block *sum_blk, block_t blk_addr)
924 {
925 update_meta_page(sbi, (void *)sum_blk, blk_addr);
926 }
927
928 static void write_current_sum_page(struct f2fs_sb_info *sbi,
929 int type, block_t blk_addr)
930 {
931 struct curseg_info *curseg = CURSEG_I(sbi, type);
932 struct page *page = grab_meta_page(sbi, blk_addr);
933 struct f2fs_summary_block *src = curseg->sum_blk;
934 struct f2fs_summary_block *dst;
935
936 dst = (struct f2fs_summary_block *)page_address(page);
937
938 mutex_lock(&curseg->curseg_mutex);
939
940 down_read(&curseg->journal_rwsem);
941 memcpy(&dst->journal, curseg->journal, SUM_JOURNAL_SIZE);
942 up_read(&curseg->journal_rwsem);
943
944 memcpy(dst->entries, src->entries, SUM_ENTRY_SIZE);
945 memcpy(&dst->footer, &src->footer, SUM_FOOTER_SIZE);
946
947 mutex_unlock(&curseg->curseg_mutex);
948
949 set_page_dirty(page);
950 f2fs_put_page(page, 1);
951 }
952
953 static int is_next_segment_free(struct f2fs_sb_info *sbi, int type)
954 {
955 struct curseg_info *curseg = CURSEG_I(sbi, type);
956 unsigned int segno = curseg->segno + 1;
957 struct free_segmap_info *free_i = FREE_I(sbi);
958
959 if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec)
960 return !test_bit(segno, free_i->free_segmap);
961 return 0;
962 }
963
964 /*
965 * Find a new segment from the free segments bitmap to right order
966 * This function should be returned with success, otherwise BUG
967 */
968 static void get_new_segment(struct f2fs_sb_info *sbi,
969 unsigned int *newseg, bool new_sec, int dir)
970 {
971 struct free_segmap_info *free_i = FREE_I(sbi);
972 unsigned int segno, secno, zoneno;
973 unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone;
974 unsigned int hint = *newseg / sbi->segs_per_sec;
975 unsigned int old_zoneno = GET_ZONENO_FROM_SEGNO(sbi, *newseg);
976 unsigned int left_start = hint;
977 bool init = true;
978 int go_left = 0;
979 int i;
980
981 spin_lock(&free_i->segmap_lock);
982
983 if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {
984 segno = find_next_zero_bit(free_i->free_segmap,
985 (hint + 1) * sbi->segs_per_sec, *newseg + 1);
986 if (segno < (hint + 1) * sbi->segs_per_sec)
987 goto got_it;
988 }
989 find_other_zone:
990 secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);
991 if (secno >= MAIN_SECS(sbi)) {
992 if (dir == ALLOC_RIGHT) {
993 secno = find_next_zero_bit(free_i->free_secmap,
994 MAIN_SECS(sbi), 0);
995 f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi));
996 } else {
997 go_left = 1;
998 left_start = hint - 1;
999 }
1000 }
1001 if (go_left == 0)
1002 goto skip_left;
1003
1004 while (test_bit(left_start, free_i->free_secmap)) {
1005 if (left_start > 0) {
1006 left_start--;
1007 continue;
1008 }
1009 left_start = find_next_zero_bit(free_i->free_secmap,
1010 MAIN_SECS(sbi), 0);
1011 f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi));
1012 break;
1013 }
1014 secno = left_start;
1015 skip_left:
1016 hint = secno;
1017 segno = secno * sbi->segs_per_sec;
1018 zoneno = secno / sbi->secs_per_zone;
1019
1020 /* give up on finding another zone */
1021 if (!init)
1022 goto got_it;
1023 if (sbi->secs_per_zone == 1)
1024 goto got_it;
1025 if (zoneno == old_zoneno)
1026 goto got_it;
1027 if (dir == ALLOC_LEFT) {
1028 if (!go_left && zoneno + 1 >= total_zones)
1029 goto got_it;
1030 if (go_left && zoneno == 0)
1031 goto got_it;
1032 }
1033 for (i = 0; i < NR_CURSEG_TYPE; i++)
1034 if (CURSEG_I(sbi, i)->zone == zoneno)
1035 break;
1036
1037 if (i < NR_CURSEG_TYPE) {
1038 /* zone is in user, try another */
1039 if (go_left)
1040 hint = zoneno * sbi->secs_per_zone - 1;
1041 else if (zoneno + 1 >= total_zones)
1042 hint = 0;
1043 else
1044 hint = (zoneno + 1) * sbi->secs_per_zone;
1045 init = false;
1046 goto find_other_zone;
1047 }
1048 got_it:
1049 /* set it as dirty segment in free segmap */
1050 f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap));
1051 __set_inuse(sbi, segno);
1052 *newseg = segno;
1053 spin_unlock(&free_i->segmap_lock);
1054 }
1055
1056 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
1057 {
1058 struct curseg_info *curseg = CURSEG_I(sbi, type);
1059 struct summary_footer *sum_footer;
1060
1061 curseg->segno = curseg->next_segno;
1062 curseg->zone = GET_ZONENO_FROM_SEGNO(sbi, curseg->segno);
1063 curseg->next_blkoff = 0;
1064 curseg->next_segno = NULL_SEGNO;
1065
1066 sum_footer = &(curseg->sum_blk->footer);
1067 memset(sum_footer, 0, sizeof(struct summary_footer));
1068 if (IS_DATASEG(type))
1069 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
1070 if (IS_NODESEG(type))
1071 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
1072 __set_sit_entry_type(sbi, type, curseg->segno, modified);
1073 }
1074
1075 /*
1076 * Allocate a current working segment.
1077 * This function always allocates a free segment in LFS manner.
1078 */
1079 static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
1080 {
1081 struct curseg_info *curseg = CURSEG_I(sbi, type);
1082 unsigned int segno = curseg->segno;
1083 int dir = ALLOC_LEFT;
1084
1085 write_sum_page(sbi, curseg->sum_blk,
1086 GET_SUM_BLOCK(sbi, segno));
1087 if (type == CURSEG_WARM_DATA || type == CURSEG_COLD_DATA)
1088 dir = ALLOC_RIGHT;
1089
1090 if (test_opt(sbi, NOHEAP))
1091 dir = ALLOC_RIGHT;
1092
1093 get_new_segment(sbi, &segno, new_sec, dir);
1094 curseg->next_segno = segno;
1095 reset_curseg(sbi, type, 1);
1096 curseg->alloc_type = LFS;
1097 }
1098
1099 static void __next_free_blkoff(struct f2fs_sb_info *sbi,
1100 struct curseg_info *seg, block_t start)
1101 {
1102 struct seg_entry *se = get_seg_entry(sbi, seg->segno);
1103 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
1104 unsigned long *target_map = SIT_I(sbi)->tmp_map;
1105 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
1106 unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
1107 int i, pos;
1108
1109 for (i = 0; i < entries; i++)
1110 target_map[i] = ckpt_map[i] | cur_map[i];
1111
1112 pos = __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);
1113
1114 seg->next_blkoff = pos;
1115 }
1116
1117 /*
1118 * If a segment is written by LFS manner, next block offset is just obtained
1119 * by increasing the current block offset. However, if a segment is written by
1120 * SSR manner, next block offset obtained by calling __next_free_blkoff
1121 */
1122 static void __refresh_next_blkoff(struct f2fs_sb_info *sbi,
1123 struct curseg_info *seg)
1124 {
1125 if (seg->alloc_type == SSR)
1126 __next_free_blkoff(sbi, seg, seg->next_blkoff + 1);
1127 else
1128 seg->next_blkoff++;
1129 }
1130
1131 /*
1132 * This function always allocates a used segment(from dirty seglist) by SSR
1133 * manner, so it should recover the existing segment information of valid blocks
1134 */
1135 static void change_curseg(struct f2fs_sb_info *sbi, int type, bool reuse)
1136 {
1137 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1138 struct curseg_info *curseg = CURSEG_I(sbi, type);
1139 unsigned int new_segno = curseg->next_segno;
1140 struct f2fs_summary_block *sum_node;
1141 struct page *sum_page;
1142
1143 write_sum_page(sbi, curseg->sum_blk,
1144 GET_SUM_BLOCK(sbi, curseg->segno));
1145 __set_test_and_inuse(sbi, new_segno);
1146
1147 mutex_lock(&dirty_i->seglist_lock);
1148 __remove_dirty_segment(sbi, new_segno, PRE);
1149 __remove_dirty_segment(sbi, new_segno, DIRTY);
1150 mutex_unlock(&dirty_i->seglist_lock);
1151
1152 reset_curseg(sbi, type, 1);
1153 curseg->alloc_type = SSR;
1154 __next_free_blkoff(sbi, curseg, 0);
1155
1156 if (reuse) {
1157 sum_page = get_sum_page(sbi, new_segno);
1158 sum_node = (struct f2fs_summary_block *)page_address(sum_page);
1159 memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);
1160 f2fs_put_page(sum_page, 1);
1161 }
1162 }
1163
1164 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type)
1165 {
1166 struct curseg_info *curseg = CURSEG_I(sbi, type);
1167 const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops;
1168
1169 if (IS_NODESEG(type) || !has_not_enough_free_secs(sbi, 0))
1170 return v_ops->get_victim(sbi,
1171 &(curseg)->next_segno, BG_GC, type, SSR);
1172
1173 /* For data segments, let's do SSR more intensively */
1174 for (; type >= CURSEG_HOT_DATA; type--)
1175 if (v_ops->get_victim(sbi, &(curseg)->next_segno,
1176 BG_GC, type, SSR))
1177 return 1;
1178 return 0;
1179 }
1180
1181 /*
1182 * flush out current segment and replace it with new segment
1183 * This function should be returned with success, otherwise BUG
1184 */
1185 static void allocate_segment_by_default(struct f2fs_sb_info *sbi,
1186 int type, bool force)
1187 {
1188 struct curseg_info *curseg = CURSEG_I(sbi, type);
1189
1190 if (force)
1191 new_curseg(sbi, type, true);
1192 else if (type == CURSEG_WARM_NODE)
1193 new_curseg(sbi, type, false);
1194 else if (curseg->alloc_type == LFS && is_next_segment_free(sbi, type))
1195 new_curseg(sbi, type, false);
1196 else if (need_SSR(sbi) && get_ssr_segment(sbi, type))
1197 change_curseg(sbi, type, true);
1198 else
1199 new_curseg(sbi, type, false);
1200
1201 stat_inc_seg_type(sbi, curseg);
1202 }
1203
1204 static void __allocate_new_segments(struct f2fs_sb_info *sbi, int type)
1205 {
1206 struct curseg_info *curseg = CURSEG_I(sbi, type);
1207 unsigned int old_segno;
1208
1209 old_segno = curseg->segno;
1210 SIT_I(sbi)->s_ops->allocate_segment(sbi, type, true);
1211 locate_dirty_segment(sbi, old_segno);
1212 }
1213
1214 void allocate_new_segments(struct f2fs_sb_info *sbi)
1215 {
1216 int i;
1217
1218 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++)
1219 __allocate_new_segments(sbi, i);
1220 }
1221
1222 static const struct segment_allocation default_salloc_ops = {
1223 .allocate_segment = allocate_segment_by_default,
1224 };
1225
1226 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
1227 {
1228 __u64 start = F2FS_BYTES_TO_BLK(range->start);
1229 __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1;
1230 unsigned int start_segno, end_segno;
1231 struct cp_control cpc;
1232 int err = 0;
1233
1234 if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize)
1235 return -EINVAL;
1236
1237 cpc.trimmed = 0;
1238 if (end <= MAIN_BLKADDR(sbi))
1239 goto out;
1240
1241 /* start/end segment number in main_area */
1242 start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start);
1243 end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 :
1244 GET_SEGNO(sbi, end);
1245 cpc.reason = CP_DISCARD;
1246 cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen));
1247
1248 /* do checkpoint to issue discard commands safely */
1249 for (; start_segno <= end_segno; start_segno = cpc.trim_end + 1) {
1250 cpc.trim_start = start_segno;
1251
1252 if (sbi->discard_blks == 0)
1253 break;
1254 else if (sbi->discard_blks < BATCHED_TRIM_BLOCKS(sbi))
1255 cpc.trim_end = end_segno;
1256 else
1257 cpc.trim_end = min_t(unsigned int,
1258 rounddown(start_segno +
1259 BATCHED_TRIM_SEGMENTS(sbi),
1260 sbi->segs_per_sec) - 1, end_segno);
1261
1262 mutex_lock(&sbi->gc_mutex);
1263 err = write_checkpoint(sbi, &cpc);
1264 mutex_unlock(&sbi->gc_mutex);
1265 }
1266 out:
1267 range->len = F2FS_BLK_TO_BYTES(cpc.trimmed);
1268 return err;
1269 }
1270
1271 static bool __has_curseg_space(struct f2fs_sb_info *sbi, int type)
1272 {
1273 struct curseg_info *curseg = CURSEG_I(sbi, type);
1274 if (curseg->next_blkoff < sbi->blocks_per_seg)
1275 return true;
1276 return false;
1277 }
1278
1279 static int __get_segment_type_2(struct page *page, enum page_type p_type)
1280 {
1281 if (p_type == DATA)
1282 return CURSEG_HOT_DATA;
1283 else
1284 return CURSEG_HOT_NODE;
1285 }
1286
1287 static int __get_segment_type_4(struct page *page, enum page_type p_type)
1288 {
1289 if (p_type == DATA) {
1290 struct inode *inode = page->mapping->host;
1291
1292 if (S_ISDIR(inode->i_mode))
1293 return CURSEG_HOT_DATA;
1294 else
1295 return CURSEG_COLD_DATA;
1296 } else {
1297 if (IS_DNODE(page) && is_cold_node(page))
1298 return CURSEG_WARM_NODE;
1299 else
1300 return CURSEG_COLD_NODE;
1301 }
1302 }
1303
1304 static int __get_segment_type_6(struct page *page, enum page_type p_type)
1305 {
1306 if (p_type == DATA) {
1307 struct inode *inode = page->mapping->host;
1308
1309 if (S_ISDIR(inode->i_mode))
1310 return CURSEG_HOT_DATA;
1311 else if (is_cold_data(page) || file_is_cold(inode))
1312 return CURSEG_COLD_DATA;
1313 else
1314 return CURSEG_WARM_DATA;
1315 } else {
1316 if (IS_DNODE(page))
1317 return is_cold_node(page) ? CURSEG_WARM_NODE :
1318 CURSEG_HOT_NODE;
1319 else
1320 return CURSEG_COLD_NODE;
1321 }
1322 }
1323
1324 static int __get_segment_type(struct page *page, enum page_type p_type)
1325 {
1326 switch (F2FS_P_SB(page)->active_logs) {
1327 case 2:
1328 return __get_segment_type_2(page, p_type);
1329 case 4:
1330 return __get_segment_type_4(page, p_type);
1331 }
1332 /* NR_CURSEG_TYPE(6) logs by default */
1333 f2fs_bug_on(F2FS_P_SB(page),
1334 F2FS_P_SB(page)->active_logs != NR_CURSEG_TYPE);
1335 return __get_segment_type_6(page, p_type);
1336 }
1337
1338 void allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
1339 block_t old_blkaddr, block_t *new_blkaddr,
1340 struct f2fs_summary *sum, int type)
1341 {
1342 struct sit_info *sit_i = SIT_I(sbi);
1343 struct curseg_info *curseg;
1344 bool direct_io = (type == CURSEG_DIRECT_IO);
1345
1346 type = direct_io ? CURSEG_WARM_DATA : type;
1347
1348 curseg = CURSEG_I(sbi, type);
1349
1350 mutex_lock(&curseg->curseg_mutex);
1351 mutex_lock(&sit_i->sentry_lock);
1352
1353 /* direct_io'ed data is aligned to the segment for better performance */
1354 if (direct_io && curseg->next_blkoff &&
1355 !has_not_enough_free_secs(sbi, 0))
1356 __allocate_new_segments(sbi, type);
1357
1358 *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
1359
1360 /*
1361 * __add_sum_entry should be resided under the curseg_mutex
1362 * because, this function updates a summary entry in the
1363 * current summary block.
1364 */
1365 __add_sum_entry(sbi, type, sum);
1366
1367 __refresh_next_blkoff(sbi, curseg);
1368
1369 stat_inc_block_count(sbi, curseg);
1370
1371 if (!__has_curseg_space(sbi, type))
1372 sit_i->s_ops->allocate_segment(sbi, type, false);
1373 /*
1374 * SIT information should be updated before segment allocation,
1375 * since SSR needs latest valid block information.
1376 */
1377 refresh_sit_entry(sbi, old_blkaddr, *new_blkaddr);
1378
1379 mutex_unlock(&sit_i->sentry_lock);
1380
1381 if (page && IS_NODESEG(type))
1382 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
1383
1384 mutex_unlock(&curseg->curseg_mutex);
1385 }
1386
1387 static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
1388 {
1389 int type = __get_segment_type(fio->page, fio->type);
1390
1391 allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr,
1392 &fio->new_blkaddr, sum, type);
1393
1394 /* writeout dirty page into bdev */
1395 f2fs_submit_page_mbio(fio);
1396 }
1397
1398 void write_meta_page(struct f2fs_sb_info *sbi, struct page *page)
1399 {
1400 struct f2fs_io_info fio = {
1401 .sbi = sbi,
1402 .type = META,
1403 .rw = WRITE_SYNC | REQ_META | REQ_PRIO,
1404 .old_blkaddr = page->index,
1405 .new_blkaddr = page->index,
1406 .page = page,
1407 .encrypted_page = NULL,
1408 };
1409
1410 if (unlikely(page->index >= MAIN_BLKADDR(sbi)))
1411 fio.rw &= ~REQ_META;
1412
1413 set_page_writeback(page);
1414 f2fs_submit_page_mbio(&fio);
1415 }
1416
1417 void write_node_page(unsigned int nid, struct f2fs_io_info *fio)
1418 {
1419 struct f2fs_summary sum;
1420
1421 set_summary(&sum, nid, 0, 0);
1422 do_write_page(&sum, fio);
1423 }
1424
1425 void write_data_page(struct dnode_of_data *dn, struct f2fs_io_info *fio)
1426 {
1427 struct f2fs_sb_info *sbi = fio->sbi;
1428 struct f2fs_summary sum;
1429 struct node_info ni;
1430
1431 f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);
1432 get_node_info(sbi, dn->nid, &ni);
1433 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
1434 do_write_page(&sum, fio);
1435 f2fs_update_data_blkaddr(dn, fio->new_blkaddr);
1436 }
1437
1438 void rewrite_data_page(struct f2fs_io_info *fio)
1439 {
1440 fio->new_blkaddr = fio->old_blkaddr;
1441 stat_inc_inplace_blocks(fio->sbi);
1442 f2fs_submit_page_mbio(fio);
1443 }
1444
1445 void __f2fs_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
1446 block_t old_blkaddr, block_t new_blkaddr,
1447 bool recover_curseg, bool recover_newaddr)
1448 {
1449 struct sit_info *sit_i = SIT_I(sbi);
1450 struct curseg_info *curseg;
1451 unsigned int segno, old_cursegno;
1452 struct seg_entry *se;
1453 int type;
1454 unsigned short old_blkoff;
1455
1456 segno = GET_SEGNO(sbi, new_blkaddr);
1457 se = get_seg_entry(sbi, segno);
1458 type = se->type;
1459
1460 if (!recover_curseg) {
1461 /* for recovery flow */
1462 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {
1463 if (old_blkaddr == NULL_ADDR)
1464 type = CURSEG_COLD_DATA;
1465 else
1466 type = CURSEG_WARM_DATA;
1467 }
1468 } else {
1469 if (!IS_CURSEG(sbi, segno))
1470 type = CURSEG_WARM_DATA;
1471 }
1472
1473 curseg = CURSEG_I(sbi, type);
1474
1475 mutex_lock(&curseg->curseg_mutex);
1476 mutex_lock(&sit_i->sentry_lock);
1477
1478 old_cursegno = curseg->segno;
1479 old_blkoff = curseg->next_blkoff;
1480
1481 /* change the current segment */
1482 if (segno != curseg->segno) {
1483 curseg->next_segno = segno;
1484 change_curseg(sbi, type, true);
1485 }
1486
1487 curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
1488 __add_sum_entry(sbi, type, sum);
1489
1490 if (!recover_curseg || recover_newaddr)
1491 update_sit_entry(sbi, new_blkaddr, 1);
1492 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
1493 update_sit_entry(sbi, old_blkaddr, -1);
1494
1495 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
1496 locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr));
1497
1498 locate_dirty_segment(sbi, old_cursegno);
1499
1500 if (recover_curseg) {
1501 if (old_cursegno != curseg->segno) {
1502 curseg->next_segno = old_cursegno;
1503 change_curseg(sbi, type, true);
1504 }
1505 curseg->next_blkoff = old_blkoff;
1506 }
1507
1508 mutex_unlock(&sit_i->sentry_lock);
1509 mutex_unlock(&curseg->curseg_mutex);
1510 }
1511
1512 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
1513 block_t old_addr, block_t new_addr,
1514 unsigned char version, bool recover_curseg,
1515 bool recover_newaddr)
1516 {
1517 struct f2fs_summary sum;
1518
1519 set_summary(&sum, dn->nid, dn->ofs_in_node, version);
1520
1521 __f2fs_replace_block(sbi, &sum, old_addr, new_addr,
1522 recover_curseg, recover_newaddr);
1523
1524 f2fs_update_data_blkaddr(dn, new_addr);
1525 }
1526
1527 void f2fs_wait_on_page_writeback(struct page *page,
1528 enum page_type type, bool ordered)
1529 {
1530 if (PageWriteback(page)) {
1531 struct f2fs_sb_info *sbi = F2FS_P_SB(page);
1532
1533 f2fs_submit_merged_bio_cond(sbi, NULL, page, 0, type, WRITE);
1534 if (ordered)
1535 wait_on_page_writeback(page);
1536 else
1537 wait_for_stable_page(page);
1538 }
1539 }
1540
1541 void f2fs_wait_on_encrypted_page_writeback(struct f2fs_sb_info *sbi,
1542 block_t blkaddr)
1543 {
1544 struct page *cpage;
1545
1546 if (blkaddr == NEW_ADDR)
1547 return;
1548
1549 f2fs_bug_on(sbi, blkaddr == NULL_ADDR);
1550
1551 cpage = find_lock_page(META_MAPPING(sbi), blkaddr);
1552 if (cpage) {
1553 f2fs_wait_on_page_writeback(cpage, DATA, true);
1554 f2fs_put_page(cpage, 1);
1555 }
1556 }
1557
1558 static int read_compacted_summaries(struct f2fs_sb_info *sbi)
1559 {
1560 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1561 struct curseg_info *seg_i;
1562 unsigned char *kaddr;
1563 struct page *page;
1564 block_t start;
1565 int i, j, offset;
1566
1567 start = start_sum_block(sbi);
1568
1569 page = get_meta_page(sbi, start++);
1570 kaddr = (unsigned char *)page_address(page);
1571
1572 /* Step 1: restore nat cache */
1573 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
1574 memcpy(seg_i->journal, kaddr, SUM_JOURNAL_SIZE);
1575
1576 /* Step 2: restore sit cache */
1577 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
1578 memcpy(seg_i->journal, kaddr + SUM_JOURNAL_SIZE, SUM_JOURNAL_SIZE);
1579 offset = 2 * SUM_JOURNAL_SIZE;
1580
1581 /* Step 3: restore summary entries */
1582 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
1583 unsigned short blk_off;
1584 unsigned int segno;
1585
1586 seg_i = CURSEG_I(sbi, i);
1587 segno = le32_to_cpu(ckpt->cur_data_segno[i]);
1588 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);
1589 seg_i->next_segno = segno;
1590 reset_curseg(sbi, i, 0);
1591 seg_i->alloc_type = ckpt->alloc_type[i];
1592 seg_i->next_blkoff = blk_off;
1593
1594 if (seg_i->alloc_type == SSR)
1595 blk_off = sbi->blocks_per_seg;
1596
1597 for (j = 0; j < blk_off; j++) {
1598 struct f2fs_summary *s;
1599 s = (struct f2fs_summary *)(kaddr + offset);
1600 seg_i->sum_blk->entries[j] = *s;
1601 offset += SUMMARY_SIZE;
1602 if (offset + SUMMARY_SIZE <= PAGE_SIZE -
1603 SUM_FOOTER_SIZE)
1604 continue;
1605
1606 f2fs_put_page(page, 1);
1607 page = NULL;
1608
1609 page = get_meta_page(sbi, start++);
1610 kaddr = (unsigned char *)page_address(page);
1611 offset = 0;
1612 }
1613 }
1614 f2fs_put_page(page, 1);
1615 return 0;
1616 }
1617
1618 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
1619 {
1620 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1621 struct f2fs_summary_block *sum;
1622 struct curseg_info *curseg;
1623 struct page *new;
1624 unsigned short blk_off;
1625 unsigned int segno = 0;
1626 block_t blk_addr = 0;
1627
1628 /* get segment number and block addr */
1629 if (IS_DATASEG(type)) {
1630 segno = le32_to_cpu(ckpt->cur_data_segno[type]);
1631 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
1632 CURSEG_HOT_DATA]);
1633 if (__exist_node_summaries(sbi))
1634 blk_addr = sum_blk_addr(sbi, NR_CURSEG_TYPE, type);
1635 else
1636 blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
1637 } else {
1638 segno = le32_to_cpu(ckpt->cur_node_segno[type -
1639 CURSEG_HOT_NODE]);
1640 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -
1641 CURSEG_HOT_NODE]);
1642 if (__exist_node_summaries(sbi))
1643 blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,
1644 type - CURSEG_HOT_NODE);
1645 else
1646 blk_addr = GET_SUM_BLOCK(sbi, segno);
1647 }
1648
1649 new = get_meta_page(sbi, blk_addr);
1650 sum = (struct f2fs_summary_block *)page_address(new);
1651
1652 if (IS_NODESEG(type)) {
1653 if (__exist_node_summaries(sbi)) {
1654 struct f2fs_summary *ns = &sum->entries[0];
1655 int i;
1656 for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {
1657 ns->version = 0;
1658 ns->ofs_in_node = 0;
1659 }
1660 } else {
1661 int err;
1662
1663 err = restore_node_summary(sbi, segno, sum);
1664 if (err) {
1665 f2fs_put_page(new, 1);
1666 return err;
1667 }
1668 }
1669 }
1670
1671 /* set uncompleted segment to curseg */
1672 curseg = CURSEG_I(sbi, type);
1673 mutex_lock(&curseg->curseg_mutex);
1674
1675 /* update journal info */
1676 down_write(&curseg->journal_rwsem);
1677 memcpy(curseg->journal, &sum->journal, SUM_JOURNAL_SIZE);
1678 up_write(&curseg->journal_rwsem);
1679
1680 memcpy(curseg->sum_blk->entries, sum->entries, SUM_ENTRY_SIZE);
1681 memcpy(&curseg->sum_blk->footer, &sum->footer, SUM_FOOTER_SIZE);
1682 curseg->next_segno = segno;
1683 reset_curseg(sbi, type, 0);
1684 curseg->alloc_type = ckpt->alloc_type[type];
1685 curseg->next_blkoff = blk_off;
1686 mutex_unlock(&curseg->curseg_mutex);
1687 f2fs_put_page(new, 1);
1688 return 0;
1689 }
1690
1691 static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
1692 {
1693 int type = CURSEG_HOT_DATA;
1694 int err;
1695
1696 if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG)) {
1697 int npages = npages_for_summary_flush(sbi, true);
1698
1699 if (npages >= 2)
1700 ra_meta_pages(sbi, start_sum_block(sbi), npages,
1701 META_CP, true);
1702
1703 /* restore for compacted data summary */
1704 if (read_compacted_summaries(sbi))
1705 return -EINVAL;
1706 type = CURSEG_HOT_NODE;
1707 }
1708
1709 if (__exist_node_summaries(sbi))
1710 ra_meta_pages(sbi, sum_blk_addr(sbi, NR_CURSEG_TYPE, type),
1711 NR_CURSEG_TYPE - type, META_CP, true);
1712
1713 for (; type <= CURSEG_COLD_NODE; type++) {
1714 err = read_normal_summaries(sbi, type);
1715 if (err)
1716 return err;
1717 }
1718
1719 return 0;
1720 }
1721
1722 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
1723 {
1724 struct page *page;
1725 unsigned char *kaddr;
1726 struct f2fs_summary *summary;
1727 struct curseg_info *seg_i;
1728 int written_size = 0;
1729 int i, j;
1730
1731 page = grab_meta_page(sbi, blkaddr++);
1732 kaddr = (unsigned char *)page_address(page);
1733
1734 /* Step 1: write nat cache */
1735 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
1736 memcpy(kaddr, seg_i->journal, SUM_JOURNAL_SIZE);
1737 written_size += SUM_JOURNAL_SIZE;
1738
1739 /* Step 2: write sit cache */
1740 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
1741 memcpy(kaddr + written_size, seg_i->journal, SUM_JOURNAL_SIZE);
1742 written_size += SUM_JOURNAL_SIZE;
1743
1744 /* Step 3: write summary entries */
1745 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
1746 unsigned short blkoff;
1747 seg_i = CURSEG_I(sbi, i);
1748 if (sbi->ckpt->alloc_type[i] == SSR)
1749 blkoff = sbi->blocks_per_seg;
1750 else
1751 blkoff = curseg_blkoff(sbi, i);
1752
1753 for (j = 0; j < blkoff; j++) {
1754 if (!page) {
1755 page = grab_meta_page(sbi, blkaddr++);
1756 kaddr = (unsigned char *)page_address(page);
1757 written_size = 0;
1758 }
1759 summary = (struct f2fs_summary *)(kaddr + written_size);
1760 *summary = seg_i->sum_blk->entries[j];
1761 written_size += SUMMARY_SIZE;
1762
1763 if (written_size + SUMMARY_SIZE <= PAGE_SIZE -
1764 SUM_FOOTER_SIZE)
1765 continue;
1766
1767 set_page_dirty(page);
1768 f2fs_put_page(page, 1);
1769 page = NULL;
1770 }
1771 }
1772 if (page) {
1773 set_page_dirty(page);
1774 f2fs_put_page(page, 1);
1775 }
1776 }
1777
1778 static void write_normal_summaries(struct f2fs_sb_info *sbi,
1779 block_t blkaddr, int type)
1780 {
1781 int i, end;
1782 if (IS_DATASEG(type))
1783 end = type + NR_CURSEG_DATA_TYPE;
1784 else
1785 end = type + NR_CURSEG_NODE_TYPE;
1786
1787 for (i = type; i < end; i++)
1788 write_current_sum_page(sbi, i, blkaddr + (i - type));
1789 }
1790
1791 void write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
1792 {
1793 if (is_set_ckpt_flags(F2FS_CKPT(sbi), CP_COMPACT_SUM_FLAG))
1794 write_compacted_summaries(sbi, start_blk);
1795 else
1796 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);
1797 }
1798
1799 void write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
1800 {
1801 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);
1802 }
1803
1804 int lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
1805 unsigned int val, int alloc)
1806 {
1807 int i;
1808
1809 if (type == NAT_JOURNAL) {
1810 for (i = 0; i < nats_in_cursum(journal); i++) {
1811 if (le32_to_cpu(nid_in_journal(journal, i)) == val)
1812 return i;
1813 }
1814 if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL))
1815 return update_nats_in_cursum(journal, 1);
1816 } else if (type == SIT_JOURNAL) {
1817 for (i = 0; i < sits_in_cursum(journal); i++)
1818 if (le32_to_cpu(segno_in_journal(journal, i)) == val)
1819 return i;
1820 if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL))
1821 return update_sits_in_cursum(journal, 1);
1822 }
1823 return -1;
1824 }
1825
1826 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
1827 unsigned int segno)
1828 {
1829 return get_meta_page(sbi, current_sit_addr(sbi, segno));
1830 }
1831
1832 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
1833 unsigned int start)
1834 {
1835 struct sit_info *sit_i = SIT_I(sbi);
1836 struct page *src_page, *dst_page;
1837 pgoff_t src_off, dst_off;
1838 void *src_addr, *dst_addr;
1839
1840 src_off = current_sit_addr(sbi, start);
1841 dst_off = next_sit_addr(sbi, src_off);
1842
1843 /* get current sit block page without lock */
1844 src_page = get_meta_page(sbi, src_off);
1845 dst_page = grab_meta_page(sbi, dst_off);
1846 f2fs_bug_on(sbi, PageDirty(src_page));
1847
1848 src_addr = page_address(src_page);
1849 dst_addr = page_address(dst_page);
1850 memcpy(dst_addr, src_addr, PAGE_SIZE);
1851
1852 set_page_dirty(dst_page);
1853 f2fs_put_page(src_page, 1);
1854
1855 set_to_next_sit(sit_i, start);
1856
1857 return dst_page;
1858 }
1859
1860 static struct sit_entry_set *grab_sit_entry_set(void)
1861 {
1862 struct sit_entry_set *ses =
1863 f2fs_kmem_cache_alloc(sit_entry_set_slab, GFP_NOFS);
1864
1865 ses->entry_cnt = 0;
1866 INIT_LIST_HEAD(&ses->set_list);
1867 return ses;
1868 }
1869
1870 static void release_sit_entry_set(struct sit_entry_set *ses)
1871 {
1872 list_del(&ses->set_list);
1873 kmem_cache_free(sit_entry_set_slab, ses);
1874 }
1875
1876 static void adjust_sit_entry_set(struct sit_entry_set *ses,
1877 struct list_head *head)
1878 {
1879 struct sit_entry_set *next = ses;
1880
1881 if (list_is_last(&ses->set_list, head))
1882 return;
1883
1884 list_for_each_entry_continue(next, head, set_list)
1885 if (ses->entry_cnt <= next->entry_cnt)
1886 break;
1887
1888 list_move_tail(&ses->set_list, &next->set_list);
1889 }
1890
1891 static void add_sit_entry(unsigned int segno, struct list_head *head)
1892 {
1893 struct sit_entry_set *ses;
1894 unsigned int start_segno = START_SEGNO(segno);
1895
1896 list_for_each_entry(ses, head, set_list) {
1897 if (ses->start_segno == start_segno) {
1898 ses->entry_cnt++;
1899 adjust_sit_entry_set(ses, head);
1900 return;
1901 }
1902 }
1903
1904 ses = grab_sit_entry_set();
1905
1906 ses->start_segno = start_segno;
1907 ses->entry_cnt++;
1908 list_add(&ses->set_list, head);
1909 }
1910
1911 static void add_sits_in_set(struct f2fs_sb_info *sbi)
1912 {
1913 struct f2fs_sm_info *sm_info = SM_I(sbi);
1914 struct list_head *set_list = &sm_info->sit_entry_set;
1915 unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap;
1916 unsigned int segno;
1917
1918 for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))
1919 add_sit_entry(segno, set_list);
1920 }
1921
1922 static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
1923 {
1924 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
1925 struct f2fs_journal *journal = curseg->journal;
1926 int i;
1927
1928 down_write(&curseg->journal_rwsem);
1929 for (i = 0; i < sits_in_cursum(journal); i++) {
1930 unsigned int segno;
1931 bool dirtied;
1932
1933 segno = le32_to_cpu(segno_in_journal(journal, i));
1934 dirtied = __mark_sit_entry_dirty(sbi, segno);
1935
1936 if (!dirtied)
1937 add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);
1938 }
1939 update_sits_in_cursum(journal, -i);
1940 up_write(&curseg->journal_rwsem);
1941 }
1942
1943 /*
1944 * CP calls this function, which flushes SIT entries including sit_journal,
1945 * and moves prefree segs to free segs.
1946 */
1947 void flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
1948 {
1949 struct sit_info *sit_i = SIT_I(sbi);
1950 unsigned long *bitmap = sit_i->dirty_sentries_bitmap;
1951 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
1952 struct f2fs_journal *journal = curseg->journal;
1953 struct sit_entry_set *ses, *tmp;
1954 struct list_head *head = &SM_I(sbi)->sit_entry_set;
1955 bool to_journal = true;
1956 struct seg_entry *se;
1957
1958 mutex_lock(&sit_i->sentry_lock);
1959
1960 if (!sit_i->dirty_sentries)
1961 goto out;
1962
1963 /*
1964 * add and account sit entries of dirty bitmap in sit entry
1965 * set temporarily
1966 */
1967 add_sits_in_set(sbi);
1968
1969 /*
1970 * if there are no enough space in journal to store dirty sit
1971 * entries, remove all entries from journal and add and account
1972 * them in sit entry set.
1973 */
1974 if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL))
1975 remove_sits_in_journal(sbi);
1976
1977 /*
1978 * there are two steps to flush sit entries:
1979 * #1, flush sit entries to journal in current cold data summary block.
1980 * #2, flush sit entries to sit page.
1981 */
1982 list_for_each_entry_safe(ses, tmp, head, set_list) {
1983 struct page *page = NULL;
1984 struct f2fs_sit_block *raw_sit = NULL;
1985 unsigned int start_segno = ses->start_segno;
1986 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK,
1987 (unsigned long)MAIN_SEGS(sbi));
1988 unsigned int segno = start_segno;
1989
1990 if (to_journal &&
1991 !__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL))
1992 to_journal = false;
1993
1994 if (to_journal) {
1995 down_write(&curseg->journal_rwsem);
1996 } else {
1997 page = get_next_sit_page(sbi, start_segno);
1998 raw_sit = page_address(page);
1999 }
2000
2001 /* flush dirty sit entries in region of current sit set */
2002 for_each_set_bit_from(segno, bitmap, end) {
2003 int offset, sit_offset;
2004
2005 se = get_seg_entry(sbi, segno);
2006
2007 /* add discard candidates */
2008 if (cpc->reason != CP_DISCARD) {
2009 cpc->trim_start = segno;
2010 add_discard_addrs(sbi, cpc);
2011 }
2012
2013 if (to_journal) {
2014 offset = lookup_journal_in_cursum(journal,
2015 SIT_JOURNAL, segno, 1);
2016 f2fs_bug_on(sbi, offset < 0);
2017 segno_in_journal(journal, offset) =
2018 cpu_to_le32(segno);
2019 seg_info_to_raw_sit(se,
2020 &sit_in_journal(journal, offset));
2021 } else {
2022 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);
2023 seg_info_to_raw_sit(se,
2024 &raw_sit->entries[sit_offset]);
2025 }
2026
2027 __clear_bit(segno, bitmap);
2028 sit_i->dirty_sentries--;
2029 ses->entry_cnt--;
2030 }
2031
2032 if (to_journal)
2033 up_write(&curseg->journal_rwsem);
2034 else
2035 f2fs_put_page(page, 1);
2036
2037 f2fs_bug_on(sbi, ses->entry_cnt);
2038 release_sit_entry_set(ses);
2039 }
2040
2041 f2fs_bug_on(sbi, !list_empty(head));
2042 f2fs_bug_on(sbi, sit_i->dirty_sentries);
2043 out:
2044 if (cpc->reason == CP_DISCARD) {
2045 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++)
2046 add_discard_addrs(sbi, cpc);
2047 }
2048 mutex_unlock(&sit_i->sentry_lock);
2049
2050 set_prefree_as_free_segments(sbi);
2051 }
2052
2053 static int build_sit_info(struct f2fs_sb_info *sbi)
2054 {
2055 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
2056 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
2057 struct sit_info *sit_i;
2058 unsigned int sit_segs, start;
2059 char *src_bitmap, *dst_bitmap;
2060 unsigned int bitmap_size;
2061
2062 /* allocate memory for SIT information */
2063 sit_i = kzalloc(sizeof(struct sit_info), GFP_KERNEL);
2064 if (!sit_i)
2065 return -ENOMEM;
2066
2067 SM_I(sbi)->sit_info = sit_i;
2068
2069 sit_i->sentries = f2fs_kvzalloc(MAIN_SEGS(sbi) *
2070 sizeof(struct seg_entry), GFP_KERNEL);
2071 if (!sit_i->sentries)
2072 return -ENOMEM;
2073
2074 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
2075 sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(bitmap_size, GFP_KERNEL);
2076 if (!sit_i->dirty_sentries_bitmap)
2077 return -ENOMEM;
2078
2079 for (start = 0; start < MAIN_SEGS(sbi); start++) {
2080 sit_i->sentries[start].cur_valid_map
2081 = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2082 sit_i->sentries[start].ckpt_valid_map
2083 = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2084 sit_i->sentries[start].discard_map
2085 = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2086 if (!sit_i->sentries[start].cur_valid_map ||
2087 !sit_i->sentries[start].ckpt_valid_map ||
2088 !sit_i->sentries[start].discard_map)
2089 return -ENOMEM;
2090 }
2091
2092 sit_i->tmp_map = kzalloc(SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
2093 if (!sit_i->tmp_map)
2094 return -ENOMEM;
2095
2096 if (sbi->segs_per_sec > 1) {
2097 sit_i->sec_entries = f2fs_kvzalloc(MAIN_SECS(sbi) *
2098 sizeof(struct sec_entry), GFP_KERNEL);
2099 if (!sit_i->sec_entries)
2100 return -ENOMEM;
2101 }
2102
2103 /* get information related with SIT */
2104 sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;
2105
2106 /* setup SIT bitmap from ckeckpoint pack */
2107 bitmap_size = __bitmap_size(sbi, SIT_BITMAP);
2108 src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);
2109
2110 dst_bitmap = kmemdup(src_bitmap, bitmap_size, GFP_KERNEL);
2111 if (!dst_bitmap)
2112 return -ENOMEM;
2113
2114 /* init SIT information */
2115 sit_i->s_ops = &default_salloc_ops;
2116
2117 sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
2118 sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;
2119 sit_i->written_valid_blocks = le64_to_cpu(ckpt->valid_block_count);
2120 sit_i->sit_bitmap = dst_bitmap;
2121 sit_i->bitmap_size = bitmap_size;
2122 sit_i->dirty_sentries = 0;
2123 sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
2124 sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
2125 sit_i->mounted_time = CURRENT_TIME_SEC.tv_sec;
2126 mutex_init(&sit_i->sentry_lock);
2127 return 0;
2128 }
2129
2130 static int build_free_segmap(struct f2fs_sb_info *sbi)
2131 {
2132 struct free_segmap_info *free_i;
2133 unsigned int bitmap_size, sec_bitmap_size;
2134
2135 /* allocate memory for free segmap information */
2136 free_i = kzalloc(sizeof(struct free_segmap_info), GFP_KERNEL);
2137 if (!free_i)
2138 return -ENOMEM;
2139
2140 SM_I(sbi)->free_info = free_i;
2141
2142 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
2143 free_i->free_segmap = f2fs_kvmalloc(bitmap_size, GFP_KERNEL);
2144 if (!free_i->free_segmap)
2145 return -ENOMEM;
2146
2147 sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
2148 free_i->free_secmap = f2fs_kvmalloc(sec_bitmap_size, GFP_KERNEL);
2149 if (!free_i->free_secmap)
2150 return -ENOMEM;
2151
2152 /* set all segments as dirty temporarily */
2153 memset(free_i->free_segmap, 0xff, bitmap_size);
2154 memset(free_i->free_secmap, 0xff, sec_bitmap_size);
2155
2156 /* init free segmap information */
2157 free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi));
2158 free_i->free_segments = 0;
2159 free_i->free_sections = 0;
2160 spin_lock_init(&free_i->segmap_lock);
2161 return 0;
2162 }
2163
2164 static int build_curseg(struct f2fs_sb_info *sbi)
2165 {
2166 struct curseg_info *array;
2167 int i;
2168
2169 array = kcalloc(NR_CURSEG_TYPE, sizeof(*array), GFP_KERNEL);
2170 if (!array)
2171 return -ENOMEM;
2172
2173 SM_I(sbi)->curseg_array = array;
2174
2175 for (i = 0; i < NR_CURSEG_TYPE; i++) {
2176 mutex_init(&array[i].curseg_mutex);
2177 array[i].sum_blk = kzalloc(PAGE_SIZE, GFP_KERNEL);
2178 if (!array[i].sum_blk)
2179 return -ENOMEM;
2180 init_rwsem(&array[i].journal_rwsem);
2181 array[i].journal = kzalloc(sizeof(struct f2fs_journal),
2182 GFP_KERNEL);
2183 if (!array[i].journal)
2184 return -ENOMEM;
2185 array[i].segno = NULL_SEGNO;
2186 array[i].next_blkoff = 0;
2187 }
2188 return restore_curseg_summaries(sbi);
2189 }
2190
2191 static void build_sit_entries(struct f2fs_sb_info *sbi)
2192 {
2193 struct sit_info *sit_i = SIT_I(sbi);
2194 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
2195 struct f2fs_journal *journal = curseg->journal;
2196 int sit_blk_cnt = SIT_BLK_CNT(sbi);
2197 unsigned int i, start, end;
2198 unsigned int readed, start_blk = 0;
2199 int nrpages = MAX_BIO_BLOCKS(sbi) * 8;
2200
2201 do {
2202 readed = ra_meta_pages(sbi, start_blk, nrpages, META_SIT, true);
2203
2204 start = start_blk * sit_i->sents_per_block;
2205 end = (start_blk + readed) * sit_i->sents_per_block;
2206
2207 for (; start < end && start < MAIN_SEGS(sbi); start++) {
2208 struct seg_entry *se = &sit_i->sentries[start];
2209 struct f2fs_sit_block *sit_blk;
2210 struct f2fs_sit_entry sit;
2211 struct page *page;
2212
2213 down_read(&curseg->journal_rwsem);
2214 for (i = 0; i < sits_in_cursum(journal); i++) {
2215 if (le32_to_cpu(segno_in_journal(journal, i))
2216 == start) {
2217 sit = sit_in_journal(journal, i);
2218 up_read(&curseg->journal_rwsem);
2219 goto got_it;
2220 }
2221 }
2222 up_read(&curseg->journal_rwsem);
2223
2224 page = get_current_sit_page(sbi, start);
2225 sit_blk = (struct f2fs_sit_block *)page_address(page);
2226 sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
2227 f2fs_put_page(page, 1);
2228 got_it:
2229 check_block_count(sbi, start, &sit);
2230 seg_info_from_raw_sit(se, &sit);
2231
2232 /* build discard map only one time */
2233 memcpy(se->discard_map, se->cur_valid_map, SIT_VBLOCK_MAP_SIZE);
2234 sbi->discard_blks += sbi->blocks_per_seg - se->valid_blocks;
2235
2236 if (sbi->segs_per_sec > 1) {
2237 struct sec_entry *e = get_sec_entry(sbi, start);
2238 e->valid_blocks += se->valid_blocks;
2239 }
2240 }
2241 start_blk += readed;
2242 } while (start_blk < sit_blk_cnt);
2243 }
2244
2245 static void init_free_segmap(struct f2fs_sb_info *sbi)
2246 {
2247 unsigned int start;
2248 int type;
2249
2250 for (start = 0; start < MAIN_SEGS(sbi); start++) {
2251 struct seg_entry *sentry = get_seg_entry(sbi, start);
2252 if (!sentry->valid_blocks)
2253 __set_free(sbi, start);
2254 }
2255
2256 /* set use the current segments */
2257 for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
2258 struct curseg_info *curseg_t = CURSEG_I(sbi, type);
2259 __set_test_and_inuse(sbi, curseg_t->segno);
2260 }
2261 }
2262
2263 static void init_dirty_segmap(struct f2fs_sb_info *sbi)
2264 {
2265 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2266 struct free_segmap_info *free_i = FREE_I(sbi);
2267 unsigned int segno = 0, offset = 0;
2268 unsigned short valid_blocks;
2269
2270 while (1) {
2271 /* find dirty segment based on free segmap */
2272 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset);
2273 if (segno >= MAIN_SEGS(sbi))
2274 break;
2275 offset = segno + 1;
2276 valid_blocks = get_valid_blocks(sbi, segno, 0);
2277 if (valid_blocks == sbi->blocks_per_seg || !valid_blocks)
2278 continue;
2279 if (valid_blocks > sbi->blocks_per_seg) {
2280 f2fs_bug_on(sbi, 1);
2281 continue;
2282 }
2283 mutex_lock(&dirty_i->seglist_lock);
2284 __locate_dirty_segment(sbi, segno, DIRTY);
2285 mutex_unlock(&dirty_i->seglist_lock);
2286 }
2287 }
2288
2289 static int init_victim_secmap(struct f2fs_sb_info *sbi)
2290 {
2291 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2292 unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
2293
2294 dirty_i->victim_secmap = f2fs_kvzalloc(bitmap_size, GFP_KERNEL);
2295 if (!dirty_i->victim_secmap)
2296 return -ENOMEM;
2297 return 0;
2298 }
2299
2300 static int build_dirty_segmap(struct f2fs_sb_info *sbi)
2301 {
2302 struct dirty_seglist_info *dirty_i;
2303 unsigned int bitmap_size, i;
2304
2305 /* allocate memory for dirty segments list information */
2306 dirty_i = kzalloc(sizeof(struct dirty_seglist_info), GFP_KERNEL);
2307 if (!dirty_i)
2308 return -ENOMEM;
2309
2310 SM_I(sbi)->dirty_info = dirty_i;
2311 mutex_init(&dirty_i->seglist_lock);
2312
2313 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
2314
2315 for (i = 0; i < NR_DIRTY_TYPE; i++) {
2316 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(bitmap_size, GFP_KERNEL);
2317 if (!dirty_i->dirty_segmap[i])
2318 return -ENOMEM;
2319 }
2320
2321 init_dirty_segmap(sbi);
2322 return init_victim_secmap(sbi);
2323 }
2324
2325 /*
2326 * Update min, max modified time for cost-benefit GC algorithm
2327 */
2328 static void init_min_max_mtime(struct f2fs_sb_info *sbi)
2329 {
2330 struct sit_info *sit_i = SIT_I(sbi);
2331 unsigned int segno;
2332
2333 mutex_lock(&sit_i->sentry_lock);
2334
2335 sit_i->min_mtime = LLONG_MAX;
2336
2337 for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
2338 unsigned int i;
2339 unsigned long long mtime = 0;
2340
2341 for (i = 0; i < sbi->segs_per_sec; i++)
2342 mtime += get_seg_entry(sbi, segno + i)->mtime;
2343
2344 mtime = div_u64(mtime, sbi->segs_per_sec);
2345
2346 if (sit_i->min_mtime > mtime)
2347 sit_i->min_mtime = mtime;
2348 }
2349 sit_i->max_mtime = get_mtime(sbi);
2350 mutex_unlock(&sit_i->sentry_lock);
2351 }
2352
2353 int build_segment_manager(struct f2fs_sb_info *sbi)
2354 {
2355 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
2356 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
2357 struct f2fs_sm_info *sm_info;
2358 int err;
2359
2360 sm_info = kzalloc(sizeof(struct f2fs_sm_info), GFP_KERNEL);
2361 if (!sm_info)
2362 return -ENOMEM;
2363
2364 /* init sm info */
2365 sbi->sm_info = sm_info;
2366 sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);
2367 sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);
2368 sm_info->segment_count = le32_to_cpu(raw_super->segment_count);
2369 sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
2370 sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
2371 sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
2372 sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
2373 sm_info->rec_prefree_segments = sm_info->main_segments *
2374 DEF_RECLAIM_PREFREE_SEGMENTS / 100;
2375 sm_info->ipu_policy = 1 << F2FS_IPU_FSYNC;
2376 sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
2377 sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS;
2378
2379 INIT_LIST_HEAD(&sm_info->discard_list);
2380 sm_info->nr_discards = 0;
2381 sm_info->max_discards = 0;
2382
2383 sm_info->trim_sections = DEF_BATCHED_TRIM_SECTIONS;
2384
2385 INIT_LIST_HEAD(&sm_info->sit_entry_set);
2386
2387 if (test_opt(sbi, FLUSH_MERGE) && !f2fs_readonly(sbi->sb)) {
2388 err = create_flush_cmd_control(sbi);
2389 if (err)
2390 return err;
2391 }
2392
2393 err = build_sit_info(sbi);
2394 if (err)
2395 return err;
2396 err = build_free_segmap(sbi);
2397 if (err)
2398 return err;
2399 err = build_curseg(sbi);
2400 if (err)
2401 return err;
2402
2403 /* reinit free segmap based on SIT */
2404 build_sit_entries(sbi);
2405
2406 init_free_segmap(sbi);
2407 err = build_dirty_segmap(sbi);
2408 if (err)
2409 return err;
2410
2411 init_min_max_mtime(sbi);
2412 return 0;
2413 }
2414
2415 static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
2416 enum dirty_type dirty_type)
2417 {
2418 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2419
2420 mutex_lock(&dirty_i->seglist_lock);
2421 kvfree(dirty_i->dirty_segmap[dirty_type]);
2422 dirty_i->nr_dirty[dirty_type] = 0;
2423 mutex_unlock(&dirty_i->seglist_lock);
2424 }
2425
2426 static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
2427 {
2428 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2429 kvfree(dirty_i->victim_secmap);
2430 }
2431
2432 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
2433 {
2434 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2435 int i;
2436
2437 if (!dirty_i)
2438 return;
2439
2440 /* discard pre-free/dirty segments list */
2441 for (i = 0; i < NR_DIRTY_TYPE; i++)
2442 discard_dirty_segmap(sbi, i);
2443
2444 destroy_victim_secmap(sbi);
2445 SM_I(sbi)->dirty_info = NULL;
2446 kfree(dirty_i);
2447 }
2448
2449 static void destroy_curseg(struct f2fs_sb_info *sbi)
2450 {
2451 struct curseg_info *array = SM_I(sbi)->curseg_array;
2452 int i;
2453
2454 if (!array)
2455 return;
2456 SM_I(sbi)->curseg_array = NULL;
2457 for (i = 0; i < NR_CURSEG_TYPE; i++) {
2458 kfree(array[i].sum_blk);
2459 kfree(array[i].journal);
2460 }
2461 kfree(array);
2462 }
2463
2464 static void destroy_free_segmap(struct f2fs_sb_info *sbi)
2465 {
2466 struct free_segmap_info *free_i = SM_I(sbi)->free_info;
2467 if (!free_i)
2468 return;
2469 SM_I(sbi)->free_info = NULL;
2470 kvfree(free_i->free_segmap);
2471 kvfree(free_i->free_secmap);
2472 kfree(free_i);
2473 }
2474
2475 static void destroy_sit_info(struct f2fs_sb_info *sbi)
2476 {
2477 struct sit_info *sit_i = SIT_I(sbi);
2478 unsigned int start;
2479
2480 if (!sit_i)
2481 return;
2482
2483 if (sit_i->sentries) {
2484 for (start = 0; start < MAIN_SEGS(sbi); start++) {
2485 kfree(sit_i->sentries[start].cur_valid_map);
2486 kfree(sit_i->sentries[start].ckpt_valid_map);
2487 kfree(sit_i->sentries[start].discard_map);
2488 }
2489 }
2490 kfree(sit_i->tmp_map);
2491
2492 kvfree(sit_i->sentries);
2493 kvfree(sit_i->sec_entries);
2494 kvfree(sit_i->dirty_sentries_bitmap);
2495
2496 SM_I(sbi)->sit_info = NULL;
2497 kfree(sit_i->sit_bitmap);
2498 kfree(sit_i);
2499 }
2500
2501 void destroy_segment_manager(struct f2fs_sb_info *sbi)
2502 {
2503 struct f2fs_sm_info *sm_info = SM_I(sbi);
2504
2505 if (!sm_info)
2506 return;
2507 destroy_flush_cmd_control(sbi);
2508 destroy_dirty_segmap(sbi);
2509 destroy_curseg(sbi);
2510 destroy_free_segmap(sbi);
2511 destroy_sit_info(sbi);
2512 sbi->sm_info = NULL;
2513 kfree(sm_info);
2514 }
2515
2516 int __init create_segment_manager_caches(void)
2517 {
2518 discard_entry_slab = f2fs_kmem_cache_create("discard_entry",
2519 sizeof(struct discard_entry));
2520 if (!discard_entry_slab)
2521 goto fail;
2522
2523 sit_entry_set_slab = f2fs_kmem_cache_create("sit_entry_set",
2524 sizeof(struct sit_entry_set));
2525 if (!sit_entry_set_slab)
2526 goto destory_discard_entry;
2527
2528 inmem_entry_slab = f2fs_kmem_cache_create("inmem_page_entry",
2529 sizeof(struct inmem_pages));
2530 if (!inmem_entry_slab)
2531 goto destroy_sit_entry_set;
2532 return 0;
2533
2534 destroy_sit_entry_set:
2535 kmem_cache_destroy(sit_entry_set_slab);
2536 destory_discard_entry:
2537 kmem_cache_destroy(discard_entry_slab);
2538 fail:
2539 return -ENOMEM;
2540 }
2541
2542 void destroy_segment_manager_caches(void)
2543 {
2544 kmem_cache_destroy(sit_entry_set_slab);
2545 kmem_cache_destroy(discard_entry_slab);
2546 kmem_cache_destroy(inmem_entry_slab);
2547 }
Linux kernel - Deliverables
This page took 0.120564 seconds and 4 git commands to generate.