mm/swap.c

   1 /*
   2  *  linux/mm/swap.c
   3  *
   4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
   5  */
   6
   7 /*
   8  * This file contains the default values for the operation of the
   9  * Linux VM subsystem. Fine-tuning documentation can be found in
  10  * Documentation/sysctl/vm.txt.
  11  * Started 18.12.91
  12  * Swap aging added 23.2.95, Stephen Tweedie.
  13  * Buffermem limits added 12.3.98, Rik van Riel.
  14  */
  15
  16 #include <linux/mm.h>
  17 #include <linux/sched.h>
  18 #include <linux/kernel_stat.h>
  19 #include <linux/swap.h>
  20 #include <linux/mman.h>
  21 #include <linux/pagemap.h>
  22 #include <linux/pagevec.h>
  23 #include <linux/init.h>
  24 #include <linux/export.h>
  25 #include <linux/mm_inline.h>
  26 #include <linux/percpu_counter.h>
  27 #include <linux/percpu.h>
  28 #include <linux/cpu.h>
  29 #include <linux/notifier.h>
  30 #include <linux/backing-dev.h>
  31 #include <linux/memcontrol.h>
  32 #include <linux/gfp.h>
  33
  34 #include "internal.h"
  35
  36 /* How many pages do we try to swap or page in/out together? */
  37 int page_cluster;
  38
  39 static DEFINE_PER_CPU(struct pagevec[NR_LRU_LISTS], lru_add_pvecs);
  40 static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs);
  41 static DEFINE_PER_CPU(struct pagevec, lru_deactivate_pvecs);
  42
  43 /*
  44  * This path almost never happens for VM activity - pages are normally
  45  * freed via pagevecs.  But it gets used by networking.
  46  */
  47 static void __page_cache_release(struct page *page)
  48 {
  49         if (PageLRU(page)) {
  50                 unsigned long flags;
  51                 struct zone *zone = page_zone(page);
  52
  53                 spin_lock_irqsave(&zone->lru_lock, flags);
  54                 VM_BUG_ON(!PageLRU(page));
  55                 __ClearPageLRU(page);
  56                 del_page_from_lru(zone, page);
  57                 spin_unlock_irqrestore(&zone->lru_lock, flags);
  58         }
  59 }
  60
  61 static void __put_single_page(struct page *page)
  62 {
  63         __page_cache_release(page);
  64         free_hot_cold_page(page, 0);
  65 }
  66
  67 static void __put_compound_page(struct page *page)
  68 {
  69         compound_page_dtor *dtor;
  70
  71         __page_cache_release(page);
  72         dtor = get_compound_page_dtor(page);
  73         (*dtor)(page);
  74 }
  75
  76 static void put_compound_page(struct page *page)
  77 {
  78         if (unlikely(PageTail(page))) {
  79                 /* __split_huge_page_refcount can run under us */
  80                 struct page *page_head = compound_trans_head(page);
  81
  82                 if (likely(page != page_head &&
  83                            get_page_unless_zero(page_head))) {
  84                         unsigned long flags;
  85                         /*
  86                          * page_head wasn't a dangling pointer but it
  87                          * may not be a head page anymore by the time
  88                          * we obtain the lock. That is ok as long as it
  89                          * can't be freed from under us.
  90                          */
  91                         flags = compound_lock_irqsave(page_head);
  92                         if (unlikely(!PageTail(page))) {
  93                                 /* __split_huge_page_refcount run before us */
  94                                 compound_unlock_irqrestore(page_head, flags);
  95                                 VM_BUG_ON(PageHead(page_head));
  96                                 if (put_page_testzero(page_head))
  97                                         __put_single_page(page_head);
  98                         out_put_single:
  99                                 if (put_page_testzero(page))
 100                                         __put_single_page(page);
 101                                 return;
 102                         }
 103                         VM_BUG_ON(page_head != page->first_page);
 104                         /*
 105                          * We can release the refcount taken by
 106                          * get_page_unless_zero() now that
 107                          * __split_huge_page_refcount() is blocked on
 108                          * the compound_lock.
 109                          */
 110                         if (put_page_testzero(page_head))
 111                                 VM_BUG_ON(1);
 112                         /* __split_huge_page_refcount will wait now */
 113                         VM_BUG_ON(page_mapcount(page) <= 0);
 114                         atomic_dec(&page->_mapcount);
 115                         VM_BUG_ON(atomic_read(&page_head->_count) <= 0);
 116                         VM_BUG_ON(atomic_read(&page->_count) != 0);
 117                         compound_unlock_irqrestore(page_head, flags);
 118                         if (put_page_testzero(page_head)) {
 119                                 if (PageHead(page_head))
 120                                         __put_compound_page(page_head);
 121                                 else
 122                                         __put_single_page(page_head);
 123                         }
 124                 } else {
 125                         /* page_head is a dangling pointer */
 126                         VM_BUG_ON(PageTail(page));
 127                         goto out_put_single;
 128                 }
 129         } else if (put_page_testzero(page)) {
 130                 if (PageHead(page))
 131                         __put_compound_page(page);
 132                 else
 133                         __put_single_page(page);
 134         }
 135 }
 136
 137 void put_page(struct page *page)
 138 {
 139         if (unlikely(PageCompound(page)))
 140                 put_compound_page(page);
 141         else if (put_page_testzero(page))
 142                 __put_single_page(page);
 143 }
 144 EXPORT_SYMBOL(put_page);
 145
 146 /*
 147  * This function is exported but must not be called by anything other
 148  * than get_page(). It implements the slow path of get_page().
 149  */
 150 bool __get_page_tail(struct page *page)
 151 {
 152         /*
 153          * This takes care of get_page() if run on a tail page
 154          * returned by one of the get_user_pages/follow_page variants.
 155          * get_user_pages/follow_page itself doesn't need the compound
 156          * lock because it runs __get_page_tail_foll() under the
 157          * proper PT lock that already serializes against
 158          * split_huge_page().
 159          */
 160         unsigned long flags;
 161         bool got = false;
 162         struct page *page_head = compound_trans_head(page);
 163
 164         if (likely(page != page_head && get_page_unless_zero(page_head))) {
 165                 /*
 166                  * page_head wasn't a dangling pointer but it
 167                  * may not be a head page anymore by the time
 168                  * we obtain the lock. That is ok as long as it
 169                  * can't be freed from under us.
 170                  */
 171                 flags = compound_lock_irqsave(page_head);
 172                 /* here __split_huge_page_refcount won't run anymore */
 173                 if (likely(PageTail(page))) {
 174                         __get_page_tail_foll(page, false);
 175                         got = true;
 176                 }
 177                 compound_unlock_irqrestore(page_head, flags);
 178                 if (unlikely(!got))
 179                         put_page(page_head);
 180         }
 181         return got;
 182 }
 183 EXPORT_SYMBOL(__get_page_tail);
 184
 185 /**
 186  * put_pages_list() - release a list of pages
 187  * @pages: list of pages threaded on page->lru
 188  *
 189  * Release a list of pages which are strung together on page.lru.  Currently
 190  * used by read_cache_pages() and related error recovery code.
 191  */
 192 void put_pages_list(struct list_head *pages)
 193 {
 194         while (!list_empty(pages)) {
 195                 struct page *victim;
 196
 197                 victim = list_entry(pages->prev, struct page, lru);
 198                 list_del(&victim->lru);
 199                 page_cache_release(victim);
 200         }
 201 }
 202 EXPORT_SYMBOL(put_pages_list);
 203
 204 static void pagevec_lru_move_fn(struct pagevec *pvec,
 205                                 void (*move_fn)(struct page *page, void *arg),
 206                                 void *arg)
 207 {
 208         int i;
 209         struct zone *zone = NULL;
 210         unsigned long flags = 0;
 211
 212         for (i = 0; i < pagevec_count(pvec); i++) {
 213                 struct page *page = pvec->pages[i];
 214                 struct zone *pagezone = page_zone(page);
 215
 216                 if (pagezone != zone) {
 217                         if (zone)
 218                                 spin_unlock_irqrestore(&zone->lru_lock, flags);
 219                         zone = pagezone;
 220                         spin_lock_irqsave(&zone->lru_lock, flags);
 221                 }
 222
 223                 (*move_fn)(page, arg);
 224         }
 225         if (zone)
 226                 spin_unlock_irqrestore(&zone->lru_lock, flags);
 227         release_pages(pvec->pages, pvec->nr, pvec->cold);
 228         pagevec_reinit(pvec);
 229 }
 230
 231 static void pagevec_move_tail_fn(struct page *page, void *arg)
 232 {
 233         int *pgmoved = arg;
 234
 235         if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
 236                 enum lru_list lru = page_lru_base_type(page);
 237                 struct lruvec *lruvec;
 238
 239                 lruvec = mem_cgroup_lru_move_lists(page_zone(page),
 240                                                    page, lru, lru);
 241                 list_move_tail(&page->lru, &lruvec->lists[lru]);
 242                 (*pgmoved)++;
 243         }
 244 }
 245
 246 /*
 247  * pagevec_move_tail() must be called with IRQ disabled.
 248  * Otherwise this may cause nasty races.
 249  */
 250 static void pagevec_move_tail(struct pagevec *pvec)
 251 {
 252         int pgmoved = 0;
 253
 254         pagevec_lru_move_fn(pvec, pagevec_move_tail_fn, &pgmoved);
 255         __count_vm_events(PGROTATED, pgmoved);
 256 }
 257
 258 /*
 259  * Writeback is about to end against a page which has been marked for immediate
 260  * reclaim.  If it still appears to be reclaimable, move it to the tail of the
 261  * inactive list.
 262  */
 263 void rotate_reclaimable_page(struct page *page)
 264 {
 265         if (!PageLocked(page) && !PageDirty(page) && !PageActive(page) &&
 266             !PageUnevictable(page) && PageLRU(page)) {
 267                 struct pagevec *pvec;
 268                 unsigned long flags;
 269
 270                 page_cache_get(page);
 271                 local_irq_save(flags);
 272                 pvec = &__get_cpu_var(lru_rotate_pvecs);
 273                 if (!pagevec_add(pvec, page))
 274                         pagevec_move_tail(pvec);
 275                 local_irq_restore(flags);
 276         }
 277 }
 278
 279 static void update_page_reclaim_stat(struct zone *zone, struct page *page,
 280                                      int file, int rotated)
 281 {
 282         struct zone_reclaim_stat *reclaim_stat = &zone->reclaim_stat;
 283         struct zone_reclaim_stat *memcg_reclaim_stat;
 284
 285         memcg_reclaim_stat = mem_cgroup_get_reclaim_stat_from_page(page);
 286
 287         reclaim_stat->recent_scanned[file]++;
 288         if (rotated)
 289                 reclaim_stat->recent_rotated[file]++;
 290
 291         if (!memcg_reclaim_stat)
 292                 return;
 293
 294         memcg_reclaim_stat->recent_scanned[file]++;
 295         if (rotated)
 296                 memcg_reclaim_stat->recent_rotated[file]++;
 297 }
 298
 299 static void __activate_page(struct page *page, void *arg)
 300 {
 301         struct zone *zone = page_zone(page);
 302
 303         if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
 304                 int file = page_is_file_cache(page);
 305                 int lru = page_lru_base_type(page);
 306                 del_page_from_lru_list(zone, page, lru);
 307
 308                 SetPageActive(page);
 309                 lru += LRU_ACTIVE;
 310                 add_page_to_lru_list(zone, page, lru);
 311                 __count_vm_event(PGACTIVATE);
 312
 313                 update_page_reclaim_stat(zone, page, file, 1);
 314         }
 315 }
 316
 317 #ifdef CONFIG_SMP
 318 static DEFINE_PER_CPU(struct pagevec, activate_page_pvecs);
 319
 320 static void activate_page_drain(int cpu)
 321 {
 322         struct pagevec *pvec = &per_cpu(activate_page_pvecs, cpu);
 323
 324         if (pagevec_count(pvec))
 325                 pagevec_lru_move_fn(pvec, __activate_page, NULL);
 326 }
 327
 328 void activate_page(struct page *page)
 329 {
 330         if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) {
 331                 struct pagevec *pvec = &get_cpu_var(activate_page_pvecs);
 332
 333                 page_cache_get(page);
 334                 if (!pagevec_add(pvec, page))
 335                         pagevec_lru_move_fn(pvec, __activate_page, NULL);
 336                 put_cpu_var(activate_page_pvecs);
 337         }
 338 }
 339
 340 #else
 341 static inline void activate_page_drain(int cpu)
 342 {
 343 }
 344
 345 void activate_page(struct page *page)
 346 {
 347         struct zone *zone = page_zone(page);
 348
 349         spin_lock_irq(&zone->lru_lock);
 350         __activate_page(page, NULL);
 351         spin_unlock_irq(&zone->lru_lock);
 352 }
 353 #endif
 354
 355 /*
 356  * Mark a page as having seen activity.
 357  *
 358  * inactive,unreferenced        ->      inactive,referenced
 359  * inactive,referenced          ->      active,unreferenced
 360  * active,unreferenced          ->      active,referenced
 361  */
 362 void mark_page_accessed(struct page *page)
 363 {
 364         if (!PageActive(page) && !PageUnevictable(page) &&
 365                         PageReferenced(page) && PageLRU(page)) {
 366                 activate_page(page);
 367                 ClearPageReferenced(page);
 368         } else if (!PageReferenced(page)) {
 369                 SetPageReferenced(page);
 370         }
 371 }
 372
 373 EXPORT_SYMBOL(mark_page_accessed);
 374
 375 void __lru_cache_add(struct page *page, enum lru_list lru)
 376 {
 377         struct pagevec *pvec = &get_cpu_var(lru_add_pvecs)[lru];
 378
 379         page_cache_get(page);
 380         if (!pagevec_add(pvec, page))
 381                 ____pagevec_lru_add(pvec, lru);
 382         put_cpu_var(lru_add_pvecs);
 383 }
 384 EXPORT_SYMBOL(__lru_cache_add);
 385
 386 /**
 387  * lru_cache_add_lru - add a page to a page list
 388  * @page: the page to be added to the LRU.
 389  * @lru: the LRU list to which the page is added.
 390  */
 391 void lru_cache_add_lru(struct page *page, enum lru_list lru)
 392 {
 393         if (PageActive(page)) {
 394                 VM_BUG_ON(PageUnevictable(page));
 395                 ClearPageActive(page);
 396         } else if (PageUnevictable(page)) {
 397                 VM_BUG_ON(PageActive(page));
 398                 ClearPageUnevictable(page);
 399         }
 400
 401         VM_BUG_ON(PageLRU(page) || PageActive(page) || PageUnevictable(page));
 402         __lru_cache_add(page, lru);
 403 }
 404
 405 /**
 406  * add_page_to_unevictable_list - add a page to the unevictable list
 407  * @page:  the page to be added to the unevictable list
 408  *
 409  * Add page directly to its zone's unevictable list.  To avoid races with
 410  * tasks that might be making the page evictable, through eg. munlock,
 411  * munmap or exit, while it's not on the lru, we want to add the page
 412  * while it's locked or otherwise "invisible" to other tasks.  This is
 413  * difficult to do when using the pagevec cache, so bypass that.
 414  */
 415 void add_page_to_unevictable_list(struct page *page)
 416 {
 417         struct zone *zone = page_zone(page);
 418
 419         spin_lock_irq(&zone->lru_lock);
 420         SetPageUnevictable(page);
 421         SetPageLRU(page);
 422         add_page_to_lru_list(zone, page, LRU_UNEVICTABLE);
 423         spin_unlock_irq(&zone->lru_lock);
 424 }
 425
 426 /*
 427  * If the page can not be invalidated, it is moved to the
 428  * inactive list to speed up its reclaim.  It is moved to the
 429  * head of the list, rather than the tail, to give the flusher
 430  * threads some time to write it out, as this is much more
 431  * effective than the single-page writeout from reclaim.
 432  *
 433  * If the page isn't page_mapped and dirty/writeback, the page
 434  * could reclaim asap using PG_reclaim.
 435  *
 436  * 1. active, mapped page -> none
 437  * 2. active, dirty/writeback page -> inactive, head, PG_reclaim
 438  * 3. inactive, mapped page -> none
 439  * 4. inactive, dirty/writeback page -> inactive, head, PG_reclaim
 440  * 5. inactive, clean -> inactive, tail
 441  * 6. Others -> none
 442  *
 443  * In 4, why it moves inactive's head, the VM expects the page would
 444  * be write it out by flusher threads as this is much more effective
 445  * than the single-page writeout from reclaim.
 446  */
 447 static void lru_deactivate_fn(struct page *page, void *arg)
 448 {
 449         int lru, file;
 450         bool active;
 451         struct zone *zone = page_zone(page);
 452
 453         if (!PageLRU(page))
 454                 return;
 455
 456         if (PageUnevictable(page))
 457                 return;
 458
 459         /* Some processes are using the page */
 460         if (page_mapped(page))
 461                 return;
 462
 463         active = PageActive(page);
 464
 465         file = page_is_file_cache(page);
 466         lru = page_lru_base_type(page);
 467         del_page_from_lru_list(zone, page, lru + active);
 468         ClearPageActive(page);
 469         ClearPageReferenced(page);
 470         add_page_to_lru_list(zone, page, lru);
 471
 472         if (PageWriteback(page) || PageDirty(page)) {
 473                 /*
 474                  * PG_reclaim could be raced with end_page_writeback
 475                  * It can make readahead confusing.  But race window
 476                  * is _really_ small and  it's non-critical problem.
 477                  */
 478                 SetPageReclaim(page);
 479         } else {
 480                 struct lruvec *lruvec;
 481                 /*
 482                  * The page's writeback ends up during pagevec
 483                  * We moves tha page into tail of inactive.
 484                  */
 485                 lruvec = mem_cgroup_lru_move_lists(zone, page, lru, lru);
 486                 list_move_tail(&page->lru, &lruvec->lists[lru]);
 487                 __count_vm_event(PGROTATED);
 488         }
 489
 490         if (active)
 491                 __count_vm_event(PGDEACTIVATE);
 492         update_page_reclaim_stat(zone, page, file, 0);
 493 }
 494
 495 /*
 496  * Drain pages out of the cpu's pagevecs.
 497  * Either "cpu" is the current CPU, and preemption has already been
 498  * disabled; or "cpu" is being hot-unplugged, and is already dead.
 499  */
 500 static void drain_cpu_pagevecs(int cpu)
 501 {
 502         struct pagevec *pvecs = per_cpu(lru_add_pvecs, cpu);
 503         struct pagevec *pvec;
 504         int lru;
 505
 506         for_each_lru(lru) {
 507                 pvec = &pvecs[lru - LRU_BASE];
 508                 if (pagevec_count(pvec))
 509                         ____pagevec_lru_add(pvec, lru);
 510         }
 511
 512         pvec = &per_cpu(lru_rotate_pvecs, cpu);
 513         if (pagevec_count(pvec)) {
 514                 unsigned long flags;
 515
 516                 /* No harm done if a racing interrupt already did this */
 517                 local_irq_save(flags);
 518                 pagevec_move_tail(pvec);
 519                 local_irq_restore(flags);
 520         }
 521
 522         pvec = &per_cpu(lru_deactivate_pvecs, cpu);
 523         if (pagevec_count(pvec))
 524                 pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
 525
 526         activate_page_drain(cpu);
 527 }
 528
 529 /**
 530  * deactivate_page - forcefully deactivate a page
 531  * @page: page to deactivate
 532  *
 533  * This function hints the VM that @page is a good reclaim candidate,
 534  * for example if its invalidation fails due to the page being dirty
 535  * or under writeback.
 536  */
 537 void deactivate_page(struct page *page)
 538 {
 539         /*
 540          * In a workload with many unevictable page such as mprotect, unevictable
 541          * page deactivation for accelerating reclaim is pointless.
 542          */
 543         if (PageUnevictable(page))
 544                 return;
 545
 546         if (likely(get_page_unless_zero(page))) {
 547                 struct pagevec *pvec = &get_cpu_var(lru_deactivate_pvecs);
 548
 549                 if (!pagevec_add(pvec, page))
 550                         pagevec_lru_move_fn(pvec, lru_deactivate_fn, NULL);
 551                 put_cpu_var(lru_deactivate_pvecs);
 552         }
 553 }
 554
 555 void lru_add_drain(void)
 556 {
 557         drain_cpu_pagevecs(get_cpu());
 558         put_cpu();
 559 }
 560
 561 static void lru_add_drain_per_cpu(struct work_struct *dummy)
 562 {
 563         lru_add_drain();
 564 }
 565
 566 /*
 567  * Returns 0 for success
 568  */
 569 int lru_add_drain_all(void)
 570 {
 571         return schedule_on_each_cpu(lru_add_drain_per_cpu);
 572 }
 573
 574 /*
 575  * Batched page_cache_release().  Decrement the reference count on all the
 576  * passed pages.  If it fell to zero then remove the page from the LRU and
 577  * free it.
 578  *
 579  * Avoid taking zone->lru_lock if possible, but if it is taken, retain it
 580  * for the remainder of the operation.
 581  *
 582  * The locking in this function is against shrink_inactive_list(): we recheck
 583  * the page count inside the lock to see whether shrink_inactive_list()
 584  * grabbed the page via the LRU.  If it did, give up: shrink_inactive_list()
 585  * will free it.
 586  */
 587 void release_pages(struct page **pages, int nr, int cold)
 588 {
 589         int i;
 590         LIST_HEAD(pages_to_free);
 591         struct zone *zone = NULL;
 592         unsigned long uninitialized_var(flags);
 593
 594         for (i = 0; i < nr; i++) {
 595                 struct page *page = pages[i];
 596
 597                 if (unlikely(PageCompound(page))) {
 598                         if (zone) {
 599                                 spin_unlock_irqrestore(&zone->lru_lock, flags);
 600                                 zone = NULL;
 601                         }
 602                         put_compound_page(page);
 603                         continue;
 604                 }
 605
 606                 if (!put_page_testzero(page))
 607                         continue;
 608
 609                 if (PageLRU(page)) {
 610                         struct zone *pagezone = page_zone(page);
 611
 612                         if (pagezone != zone) {
 613                                 if (zone)
 614                                         spin_unlock_irqrestore(&zone->lru_lock,
 615                                                                         flags);
 616                                 zone = pagezone;
 617                                 spin_lock_irqsave(&zone->lru_lock, flags);
 618                         }
 619                         VM_BUG_ON(!PageLRU(page));
 620                         __ClearPageLRU(page);
 621                         del_page_from_lru(zone, page);
 622                 }
 623
 624                 list_add(&page->lru, &pages_to_free);
 625         }
 626         if (zone)
 627                 spin_unlock_irqrestore(&zone->lru_lock, flags);
 628
 629         free_hot_cold_page_list(&pages_to_free, cold);
 630 }
 631 EXPORT_SYMBOL(release_pages);
 632
 633 /*
 634  * The pages which we're about to release may be in the deferred lru-addition
 635  * queues.  That would prevent them from really being freed right now.  That's
 636  * OK from a correctness point of view but is inefficient - those pages may be
 637  * cache-warm and we want to give them back to the page allocator ASAP.
 638  *
 639  * So __pagevec_release() will drain those queues here.  __pagevec_lru_add()
 640  * and __pagevec_lru_add_active() call release_pages() directly to avoid
 641  * mutual recursion.
 642  */
 643 void __pagevec_release(struct pagevec *pvec)
 644 {
 645         lru_add_drain();
 646         release_pages(pvec->pages, pagevec_count(pvec), pvec->cold);
 647         pagevec_reinit(pvec);
 648 }
 649
 650 EXPORT_SYMBOL(__pagevec_release);
 651
 652 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 653 /* used by __split_huge_page_refcount() */
 654 void lru_add_page_tail(struct zone* zone,
 655                        struct page *page, struct page *page_tail)
 656 {
 657         int active;
 658         enum lru_list lru;
 659         const int file = 0;
 660
 661         VM_BUG_ON(!PageHead(page));
 662         VM_BUG_ON(PageCompound(page_tail));
 663         VM_BUG_ON(PageLRU(page_tail));
 664         VM_BUG_ON(!spin_is_locked(&zone->lru_lock));
 665
 666         SetPageLRU(page_tail);
 667
 668         if (page_evictable(page_tail, NULL)) {
 669                 if (PageActive(page)) {
 670                         SetPageActive(page_tail);
 671                         active = 1;
 672                         lru = LRU_ACTIVE_ANON;
 673                 } else {
 674                         active = 0;
 675                         lru = LRU_INACTIVE_ANON;
 676                 }
 677                 update_page_reclaim_stat(zone, page_tail, file, active);
 678         } else {
 679                 SetPageUnevictable(page_tail);
 680                 lru = LRU_UNEVICTABLE;
 681         }
 682
 683         if (likely(PageLRU(page)))
 684                 list_add_tail(&page_tail->lru, &page->lru);
 685         else {
 686                 struct list_head *list_head;
 687                 /*
 688                  * Head page has not yet been counted, as an hpage,
 689                  * so we must account for each subpage individually.
 690                  *
 691                  * Use the standard add function to put page_tail on the list,
 692                  * but then correct its position so they all end up in order.
 693                  */
 694                 add_page_to_lru_list(zone, page_tail, lru);
 695                 list_head = page_tail->lru.prev;
 696                 list_move_tail(&page_tail->lru, list_head);
 697         }
 698 }
 699 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 700
 701 static void ____pagevec_lru_add_fn(struct page *page, void *arg)
 702 {
 703         enum lru_list lru = (enum lru_list)arg;
 704         struct zone *zone = page_zone(page);
 705         int file = is_file_lru(lru);
 706         int active = is_active_lru(lru);
 707
 708         VM_BUG_ON(PageActive(page));
 709         VM_BUG_ON(PageUnevictable(page));
 710         VM_BUG_ON(PageLRU(page));
 711
 712         SetPageLRU(page);
 713         if (active)
 714                 SetPageActive(page);
 715         update_page_reclaim_stat(zone, page, file, active);
 716         add_page_to_lru_list(zone, page, lru);
 717 }
 718
 719 /*
 720  * Add the passed pages to the LRU, then drop the caller's refcount
 721  * on them.  Reinitialises the caller's pagevec.
 722  */
 723 void ____pagevec_lru_add(struct pagevec *pvec, enum lru_list lru)
 724 {
 725         VM_BUG_ON(is_unevictable_lru(lru));
 726
 727         pagevec_lru_move_fn(pvec, ____pagevec_lru_add_fn, (void *)lru);
 728 }
 729
 730 EXPORT_SYMBOL(____pagevec_lru_add);
 731
 732 /**
 733  * pagevec_lookup - gang pagecache lookup
 734  * @pvec:       Where the resulting pages are placed
 735  * @mapping:    The address_space to search
 736  * @start:      The starting page index
 737  * @nr_pages:   The maximum number of pages
 738  *
 739  * pagevec_lookup() will search for and return a group of up to @nr_pages pages
 740  * in the mapping.  The pages are placed in @pvec.  pagevec_lookup() takes a
 741  * reference against the pages in @pvec.
 742  *
 743  * The search returns a group of mapping-contiguous pages with ascending
 744  * indexes.  There may be holes in the indices due to not-present pages.
 745  *
 746  * pagevec_lookup() returns the number of pages which were found.
 747  */
 748 unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping,
 749                 pgoff_t start, unsigned nr_pages)
 750 {
 751         pvec->nr = find_get_pages(mapping, start, nr_pages, pvec->pages);
 752         return pagevec_count(pvec);
 753 }
 754
 755 EXPORT_SYMBOL(pagevec_lookup);
 756
 757 unsigned pagevec_lookup_tag(struct pagevec *pvec, struct address_space *mapping,
 758                 pgoff_t *index, int tag, unsigned nr_pages)
 759 {
 760         pvec->nr = find_get_pages_tag(mapping, index, tag,
 761                                         nr_pages, pvec->pages);
 762         return pagevec_count(pvec);
 763 }
 764
 765 EXPORT_SYMBOL(pagevec_lookup_tag);
 766
 767 /*
 768  * Perform any setup for the swap system
 769  */
 770 void __init swap_setup(void)
 771 {
 772         unsigned long megs = totalram_pages >> (20 - PAGE_SHIFT);
 773
 774 #ifdef CONFIG_SWAP
 775         bdi_init(swapper_space.backing_dev_info);
 776 #endif
 777
 778         /* Use a smaller cluster for small-memory machines */
 779         if (megs < 16)
 780                 page_cluster = 2;
 781         else
 782                 page_cluster = 3;
 783         /*
 784          * Right now other parts of the system means that we
 785          * _really_ don't want to cluster much more
 786          */
 787 }