2 * linux/mm/swap_state.c
4 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
5 * Swap reorganised 29.12.95, Stephen Tweedie
7 * Rewritten to use page cache, (C) 1998 Stephen Tweedie
10 #include <linux/gfp.h>
11 #include <linux/kernel_stat.h>
12 #include <linux/swap.h>
13 #include <linux/swapops.h>
14 #include <linux/init.h>
15 #include <linux/pagemap.h>
16 #include <linux/backing-dev.h>
17 #include <linux/blkdev.h>
18 #include <linux/pagevec.h>
19 #include <linux/migrate.h>
21 #include <asm/pgtable.h>
24 * swapper_space is a fiction, retained to simplify the path through
25 * vmscan's shrink_page_list.
27 static const struct address_space_operations swap_aops
= {
28 .writepage
= swap_writepage
,
29 .set_page_dirty
= swap_set_page_dirty
,
30 #ifdef CONFIG_MIGRATION
31 .migratepage
= migrate_page
,
35 struct address_space swapper_spaces
[MAX_SWAPFILES
] = {
36 [0 ... MAX_SWAPFILES
- 1] = {
37 .page_tree
= RADIX_TREE_INIT(GFP_ATOMIC
|__GFP_NOWARN
),
38 .i_mmap_writable
= ATOMIC_INIT(0),
43 #define INC_CACHE_INFO(x) do { swap_cache_info.x++; } while (0)
46 unsigned long add_total
;
47 unsigned long del_total
;
48 unsigned long find_success
;
49 unsigned long find_total
;
52 unsigned long total_swapcache_pages(void)
55 unsigned long ret
= 0;
57 for (i
= 0; i
< MAX_SWAPFILES
; i
++)
58 ret
+= swapper_spaces
[i
].nrpages
;
62 static atomic_t swapin_readahead_hits
= ATOMIC_INIT(4);
64 void show_swap_cache_info(void)
66 printk("%lu pages in swap cache\n", total_swapcache_pages());
67 printk("Swap cache stats: add %lu, delete %lu, find %lu/%lu\n",
68 swap_cache_info
.add_total
, swap_cache_info
.del_total
,
69 swap_cache_info
.find_success
, swap_cache_info
.find_total
);
70 printk("Free swap = %ldkB\n",
71 get_nr_swap_pages() << (PAGE_SHIFT
- 10));
72 printk("Total swap = %lukB\n", total_swap_pages
<< (PAGE_SHIFT
- 10));
76 * __add_to_swap_cache resembles add_to_page_cache_locked on swapper_space,
77 * but sets SwapCache flag and private instead of mapping and index.
79 int __add_to_swap_cache(struct page
*page
, swp_entry_t entry
)
82 struct address_space
*address_space
;
84 VM_BUG_ON_PAGE(!PageLocked(page
), page
);
85 VM_BUG_ON_PAGE(PageSwapCache(page
), page
);
86 VM_BUG_ON_PAGE(!PageSwapBacked(page
), page
);
89 SetPageSwapCache(page
);
90 set_page_private(page
, entry
.val
);
92 address_space
= swap_address_space(entry
);
93 spin_lock_irq(&address_space
->tree_lock
);
94 error
= radix_tree_insert(&address_space
->page_tree
,
97 address_space
->nrpages
++;
98 __inc_zone_page_state(page
, NR_FILE_PAGES
);
99 INC_CACHE_INFO(add_total
);
101 spin_unlock_irq(&address_space
->tree_lock
);
103 if (unlikely(error
)) {
105 * Only the context which have set SWAP_HAS_CACHE flag
106 * would call add_to_swap_cache().
107 * So add_to_swap_cache() doesn't returns -EEXIST.
109 VM_BUG_ON(error
== -EEXIST
);
110 set_page_private(page
, 0UL);
111 ClearPageSwapCache(page
);
112 page_cache_release(page
);
119 int add_to_swap_cache(struct page
*page
, swp_entry_t entry
, gfp_t gfp_mask
)
123 error
= radix_tree_maybe_preload(gfp_mask
);
125 error
= __add_to_swap_cache(page
, entry
);
126 radix_tree_preload_end();
132 * This must be called only on pages that have
133 * been verified to be in the swap cache.
135 void __delete_from_swap_cache(struct page
*page
)
138 struct address_space
*address_space
;
140 VM_BUG_ON_PAGE(!PageLocked(page
), page
);
141 VM_BUG_ON_PAGE(!PageSwapCache(page
), page
);
142 VM_BUG_ON_PAGE(PageWriteback(page
), page
);
144 entry
.val
= page_private(page
);
145 address_space
= swap_address_space(entry
);
146 radix_tree_delete(&address_space
->page_tree
, page_private(page
));
147 set_page_private(page
, 0);
148 ClearPageSwapCache(page
);
149 address_space
->nrpages
--;
150 __dec_zone_page_state(page
, NR_FILE_PAGES
);
151 INC_CACHE_INFO(del_total
);
155 * add_to_swap - allocate swap space for a page
156 * @page: page we want to move to swap
158 * Allocate swap space for the page and add the page to the
159 * swap cache. Caller needs to hold the page lock.
161 int add_to_swap(struct page
*page
, struct list_head
*list
)
166 VM_BUG_ON_PAGE(!PageLocked(page
), page
);
167 VM_BUG_ON_PAGE(!PageUptodate(page
), page
);
169 entry
= get_swap_page();
173 if (unlikely(PageTransHuge(page
)))
174 if (unlikely(split_huge_page_to_list(page
, list
))) {
175 swapcache_free(entry
);
180 * Radix-tree node allocations from PF_MEMALLOC contexts could
181 * completely exhaust the page allocator. __GFP_NOMEMALLOC
182 * stops emergency reserves from being allocated.
184 * TODO: this could cause a theoretical memory reclaim
185 * deadlock in the swap out path.
188 * Add it to the swap cache.
190 err
= add_to_swap_cache(page
, entry
,
191 __GFP_HIGH
|__GFP_NOMEMALLOC
|__GFP_NOWARN
);
195 } else { /* -ENOMEM radix-tree allocation failure */
197 * add_to_swap_cache() doesn't return -EEXIST, so we can safely
198 * clear SWAP_HAS_CACHE flag.
200 swapcache_free(entry
);
206 * This must be called only on pages that have
207 * been verified to be in the swap cache and locked.
208 * It will never put the page into the free list,
209 * the caller has a reference on the page.
211 void delete_from_swap_cache(struct page
*page
)
214 struct address_space
*address_space
;
216 entry
.val
= page_private(page
);
218 address_space
= swap_address_space(entry
);
219 spin_lock_irq(&address_space
->tree_lock
);
220 __delete_from_swap_cache(page
);
221 spin_unlock_irq(&address_space
->tree_lock
);
223 swapcache_free(entry
);
224 page_cache_release(page
);
228 * If we are the only user, then try to free up the swap cache.
230 * Its ok to check for PageSwapCache without the page lock
231 * here because we are going to recheck again inside
232 * try_to_free_swap() _with_ the lock.
235 static inline void free_swap_cache(struct page
*page
)
237 if (PageSwapCache(page
) && !page_mapped(page
) && trylock_page(page
)) {
238 try_to_free_swap(page
);
244 * Perform a free_page(), also freeing any swap cache associated with
245 * this page if it is the last user of the page.
247 void free_page_and_swap_cache(struct page
*page
)
249 free_swap_cache(page
);
250 page_cache_release(page
);
254 * Passed an array of pages, drop them all from swapcache and then release
255 * them. They are removed from the LRU and freed if this is their last use.
257 void free_pages_and_swap_cache(struct page
**pages
, int nr
)
259 struct page
**pagep
= pages
;
263 for (i
= 0; i
< nr
; i
++)
264 free_swap_cache(pagep
[i
]);
265 release_pages(pagep
, nr
, false);
269 * Lookup a swap entry in the swap cache. A found page will be returned
270 * unlocked and with its refcount incremented - we rely on the kernel
271 * lock getting page table operations atomic even if we drop the page
272 * lock before returning.
274 struct page
* lookup_swap_cache(swp_entry_t entry
)
278 page
= find_get_page(swap_address_space(entry
), entry
.val
);
281 INC_CACHE_INFO(find_success
);
282 if (TestClearPageReadahead(page
))
283 atomic_inc(&swapin_readahead_hits
);
286 INC_CACHE_INFO(find_total
);
290 struct page
*__read_swap_cache_async(swp_entry_t entry
, gfp_t gfp_mask
,
291 struct vm_area_struct
*vma
, unsigned long addr
,
292 bool *new_page_allocated
)
294 struct page
*found_page
, *new_page
= NULL
;
295 struct address_space
*swapper_space
= swap_address_space(entry
);
297 *new_page_allocated
= false;
301 * First check the swap cache. Since this is normally
302 * called after lookup_swap_cache() failed, re-calling
303 * that would confuse statistics.
305 found_page
= find_get_page(swapper_space
, entry
.val
);
310 * Get a new page to read into from swap.
313 new_page
= alloc_page_vma(gfp_mask
, vma
, addr
);
315 break; /* Out of memory */
319 * call radix_tree_preload() while we can wait.
321 err
= radix_tree_maybe_preload(gfp_mask
& GFP_KERNEL
);
326 * Swap entry may have been freed since our caller observed it.
328 err
= swapcache_prepare(entry
);
329 if (err
== -EEXIST
) {
330 radix_tree_preload_end();
332 * We might race against get_swap_page() and stumble
333 * across a SWAP_HAS_CACHE swap_map entry whose page
334 * has not been brought into the swapcache yet, while
335 * the other end is scheduled away waiting on discard
336 * I/O completion at scan_swap_map().
338 * In order to avoid turning this transitory state
339 * into a permanent loop around this -EEXIST case
340 * if !CONFIG_PREEMPT and the I/O completion happens
341 * to be waiting on the CPU waitqueue where we are now
342 * busy looping, we just conditionally invoke the
343 * scheduler here, if there are some more important
349 if (err
) { /* swp entry is obsolete ? */
350 radix_tree_preload_end();
354 /* May fail (-ENOMEM) if radix-tree node allocation failed. */
355 __SetPageLocked(new_page
);
356 SetPageSwapBacked(new_page
);
357 err
= __add_to_swap_cache(new_page
, entry
);
359 radix_tree_preload_end();
361 * Initiate read into locked page and return.
363 lru_cache_add_anon(new_page
);
364 *new_page_allocated
= true;
367 radix_tree_preload_end();
368 ClearPageSwapBacked(new_page
);
369 __ClearPageLocked(new_page
);
371 * add_to_swap_cache() doesn't return -EEXIST, so we can safely
372 * clear SWAP_HAS_CACHE flag.
374 swapcache_free(entry
);
375 } while (err
!= -ENOMEM
);
378 page_cache_release(new_page
);
383 * Locate a page of swap in physical memory, reserving swap cache space
384 * and reading the disk if it is not already cached.
385 * A failure return means that either the page allocation failed or that
386 * the swap entry is no longer in use.
388 struct page
*read_swap_cache_async(swp_entry_t entry
, gfp_t gfp_mask
,
389 struct vm_area_struct
*vma
, unsigned long addr
)
391 bool page_was_allocated
;
392 struct page
*retpage
= __read_swap_cache_async(entry
, gfp_mask
,
393 vma
, addr
, &page_was_allocated
);
395 if (page_was_allocated
)
396 swap_readpage(retpage
);
401 static unsigned long swapin_nr_pages(unsigned long offset
)
403 static unsigned long prev_offset
;
404 unsigned int pages
, max_pages
, last_ra
;
405 static atomic_t last_readahead_pages
;
407 max_pages
= 1 << READ_ONCE(page_cluster
);
412 * This heuristic has been found to work well on both sequential and
413 * random loads, swapping to hard disk or to SSD: please don't ask
414 * what the "+ 2" means, it just happens to work well, that's all.
416 pages
= atomic_xchg(&swapin_readahead_hits
, 0) + 2;
419 * We can have no readahead hits to judge by: but must not get
420 * stuck here forever, so check for an adjacent offset instead
421 * (and don't even bother to check whether swap type is same).
423 if (offset
!= prev_offset
+ 1 && offset
!= prev_offset
- 1)
425 prev_offset
= offset
;
427 unsigned int roundup
= 4;
428 while (roundup
< pages
)
433 if (pages
> max_pages
)
436 /* Don't shrink readahead too fast */
437 last_ra
= atomic_read(&last_readahead_pages
) / 2;
440 atomic_set(&last_readahead_pages
, pages
);
446 * swapin_readahead - swap in pages in hope we need them soon
447 * @entry: swap entry of this memory
448 * @gfp_mask: memory allocation flags
449 * @vma: user vma this address belongs to
450 * @addr: target address for mempolicy
452 * Returns the struct page for entry and addr, after queueing swapin.
454 * Primitive swap readahead code. We simply read an aligned block of
455 * (1 << page_cluster) entries in the swap area. This method is chosen
456 * because it doesn't cost us any seek time. We also make sure to queue
457 * the 'original' request together with the readahead ones...
459 * This has been extended to use the NUMA policies from the mm triggering
462 * Caller must hold down_read on the vma->vm_mm if vma is not NULL.
464 struct page
*swapin_readahead(swp_entry_t entry
, gfp_t gfp_mask
,
465 struct vm_area_struct
*vma
, unsigned long addr
)
468 unsigned long entry_offset
= swp_offset(entry
);
469 unsigned long offset
= entry_offset
;
470 unsigned long start_offset
, end_offset
;
472 struct blk_plug plug
;
474 mask
= swapin_nr_pages(offset
) - 1;
478 /* Read a page_cluster sized and aligned cluster around offset. */
479 start_offset
= offset
& ~mask
;
480 end_offset
= offset
| mask
;
481 if (!start_offset
) /* First page is swap header. */
484 blk_start_plug(&plug
);
485 for (offset
= start_offset
; offset
<= end_offset
; offset
++) {
486 /* Ok, do the async read-ahead now */
487 page
= read_swap_cache_async(swp_entry(swp_type(entry
), offset
),
488 gfp_mask
, vma
, addr
);
491 if (offset
!= entry_offset
)
492 SetPageReadahead(page
);
493 page_cache_release(page
);
495 blk_finish_plug(&plug
);
497 lru_add_drain(); /* Push any new pages onto the LRU now */
499 return read_swap_cache_async(entry
, gfp_mask
, vma
, addr
);