4 * Manages VM statistics
5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
8 * Copyright (C) 2006 Silicon Graphics, Inc.,
9 * Christoph Lameter <christoph@lameter.com>
13 #include <linux/err.h>
14 #include <linux/module.h>
15 #include <linux/cpu.h>
16 #include <linux/sched.h>
18 #ifdef CONFIG_VM_EVENT_COUNTERS
19 DEFINE_PER_CPU(struct vm_event_state
, vm_event_states
) = {{0}};
20 EXPORT_PER_CPU_SYMBOL(vm_event_states
);
22 static void sum_vm_events(unsigned long *ret
, cpumask_t
*cpumask
)
27 memset(ret
, 0, NR_VM_EVENT_ITEMS
* sizeof(unsigned long));
29 cpu
= first_cpu(*cpumask
);
30 while (cpu
< NR_CPUS
) {
31 struct vm_event_state
*this = &per_cpu(vm_event_states
, cpu
);
33 cpu
= next_cpu(cpu
, *cpumask
);
36 prefetch(&per_cpu(vm_event_states
, cpu
));
39 for (i
= 0; i
< NR_VM_EVENT_ITEMS
; i
++)
40 ret
[i
] += this->event
[i
];
45 * Accumulate the vm event counters across all CPUs.
46 * The result is unavoidably approximate - it can change
47 * during and after execution of this function.
49 void all_vm_events(unsigned long *ret
)
51 sum_vm_events(ret
, &cpu_online_map
);
53 EXPORT_SYMBOL_GPL(all_vm_events
);
57 * Fold the foreign cpu events into our own.
59 * This is adding to the events on one processor
60 * but keeps the global counts constant.
62 void vm_events_fold_cpu(int cpu
)
64 struct vm_event_state
*fold_state
= &per_cpu(vm_event_states
, cpu
);
67 for (i
= 0; i
< NR_VM_EVENT_ITEMS
; i
++) {
68 count_vm_events(i
, fold_state
->event
[i
]);
69 fold_state
->event
[i
] = 0;
72 #endif /* CONFIG_HOTPLUG */
74 #endif /* CONFIG_VM_EVENT_COUNTERS */
77 * Manage combined zone based / global counters
79 * vm_stat contains the global counters
81 atomic_long_t vm_stat
[NR_VM_ZONE_STAT_ITEMS
];
82 EXPORT_SYMBOL(vm_stat
);
86 static int calculate_threshold(struct zone
*zone
)
89 int mem
; /* memory in 128 MB units */
92 * The threshold scales with the number of processors and the amount
93 * of memory per zone. More memory means that we can defer updates for
94 * longer, more processors could lead to more contention.
95 * fls() is used to have a cheap way of logarithmic scaling.
97 * Some sample thresholds:
99 * Threshold Processors (fls) Zonesize fls(mem+1)
100 * ------------------------------------------------------------------
117 * 125 1024 10 8-16 GB 8
118 * 125 1024 10 16-32 GB 9
121 mem
= zone
->present_pages
>> (27 - PAGE_SHIFT
);
123 threshold
= 2 * fls(num_online_cpus()) * (1 + fls(mem
));
126 * Maximum threshold is 125
128 threshold
= min(125, threshold
);
134 * Refresh the thresholds for each zone.
136 static void refresh_zone_stat_thresholds(void)
142 for_each_zone(zone
) {
144 if (!zone
->present_pages
)
147 threshold
= calculate_threshold(zone
);
149 for_each_online_cpu(cpu
)
150 zone_pcp(zone
, cpu
)->stat_threshold
= threshold
;
155 * For use when we know that interrupts are disabled.
157 void __mod_zone_page_state(struct zone
*zone
, enum zone_stat_item item
,
160 struct per_cpu_pageset
*pcp
= zone_pcp(zone
, smp_processor_id());
161 s8
*p
= pcp
->vm_stat_diff
+ item
;
166 if (unlikely(x
> pcp
->stat_threshold
|| x
< -pcp
->stat_threshold
)) {
167 zone_page_state_add(x
, zone
, item
);
172 EXPORT_SYMBOL(__mod_zone_page_state
);
175 * For an unknown interrupt state
177 void mod_zone_page_state(struct zone
*zone
, enum zone_stat_item item
,
182 local_irq_save(flags
);
183 __mod_zone_page_state(zone
, item
, delta
);
184 local_irq_restore(flags
);
186 EXPORT_SYMBOL(mod_zone_page_state
);
189 * Optimized increment and decrement functions.
191 * These are only for a single page and therefore can take a struct page *
192 * argument instead of struct zone *. This allows the inclusion of the code
193 * generated for page_zone(page) into the optimized functions.
195 * No overflow check is necessary and therefore the differential can be
196 * incremented or decremented in place which may allow the compilers to
197 * generate better code.
198 * The increment or decrement is known and therefore one boundary check can
201 * NOTE: These functions are very performance sensitive. Change only
204 * Some processors have inc/dec instructions that are atomic vs an interrupt.
205 * However, the code must first determine the differential location in a zone
206 * based on the processor number and then inc/dec the counter. There is no
207 * guarantee without disabling preemption that the processor will not change
208 * in between and therefore the atomicity vs. interrupt cannot be exploited
209 * in a useful way here.
211 void __inc_zone_state(struct zone
*zone
, enum zone_stat_item item
)
213 struct per_cpu_pageset
*pcp
= zone_pcp(zone
, smp_processor_id());
214 s8
*p
= pcp
->vm_stat_diff
+ item
;
218 if (unlikely(*p
> pcp
->stat_threshold
)) {
219 int overstep
= pcp
->stat_threshold
/ 2;
221 zone_page_state_add(*p
+ overstep
, zone
, item
);
226 void __inc_zone_page_state(struct page
*page
, enum zone_stat_item item
)
228 __inc_zone_state(page_zone(page
), item
);
230 EXPORT_SYMBOL(__inc_zone_page_state
);
232 void __dec_zone_state(struct zone
*zone
, enum zone_stat_item item
)
234 struct per_cpu_pageset
*pcp
= zone_pcp(zone
, smp_processor_id());
235 s8
*p
= pcp
->vm_stat_diff
+ item
;
239 if (unlikely(*p
< - pcp
->stat_threshold
)) {
240 int overstep
= pcp
->stat_threshold
/ 2;
242 zone_page_state_add(*p
- overstep
, zone
, item
);
247 void __dec_zone_page_state(struct page
*page
, enum zone_stat_item item
)
249 __dec_zone_state(page_zone(page
), item
);
251 EXPORT_SYMBOL(__dec_zone_page_state
);
253 void inc_zone_state(struct zone
*zone
, enum zone_stat_item item
)
257 local_irq_save(flags
);
258 __inc_zone_state(zone
, item
);
259 local_irq_restore(flags
);
262 void inc_zone_page_state(struct page
*page
, enum zone_stat_item item
)
267 zone
= page_zone(page
);
268 local_irq_save(flags
);
269 __inc_zone_state(zone
, item
);
270 local_irq_restore(flags
);
272 EXPORT_SYMBOL(inc_zone_page_state
);
274 void dec_zone_page_state(struct page
*page
, enum zone_stat_item item
)
278 local_irq_save(flags
);
279 __dec_zone_page_state(page
, item
);
280 local_irq_restore(flags
);
282 EXPORT_SYMBOL(dec_zone_page_state
);
285 * Update the zone counters for one cpu.
287 * Note that refresh_cpu_vm_stats strives to only access
288 * node local memory. The per cpu pagesets on remote zones are placed
289 * in the memory local to the processor using that pageset. So the
290 * loop over all zones will access a series of cachelines local to
293 * The call to zone_page_state_add updates the cachelines with the
294 * statistics in the remote zone struct as well as the global cachelines
295 * with the global counters. These could cause remote node cache line
296 * bouncing and will have to be only done when necessary.
298 void refresh_cpu_vm_stats(int cpu
)
304 for_each_zone(zone
) {
305 struct per_cpu_pageset
*p
;
307 if (!populated_zone(zone
))
310 p
= zone_pcp(zone
, cpu
);
312 for (i
= 0; i
< NR_VM_ZONE_STAT_ITEMS
; i
++)
313 if (p
->vm_stat_diff
[i
]) {
314 local_irq_save(flags
);
315 zone_page_state_add(p
->vm_stat_diff
[i
],
317 p
->vm_stat_diff
[i
] = 0;
319 /* 3 seconds idle till flush */
322 local_irq_restore(flags
);
326 * Deal with draining the remote pageset of this
329 * Check if there are pages remaining in this pageset
330 * if not then there is nothing to expire.
332 if (!p
->expire
|| (!p
->pcp
[0].count
&& !p
->pcp
[1].count
))
336 * We never drain zones local to this processor.
338 if (zone_to_nid(zone
) == numa_node_id()) {
348 drain_zone_pages(zone
, p
->pcp
+ 0);
351 drain_zone_pages(zone
, p
->pcp
+ 1);
356 static void __refresh_cpu_vm_stats(void *dummy
)
358 refresh_cpu_vm_stats(smp_processor_id());
362 * Consolidate all counters.
364 * Note that the result is less inaccurate but still inaccurate
365 * if concurrent processes are allowed to run.
367 void refresh_vm_stats(void)
369 on_each_cpu(__refresh_cpu_vm_stats
, NULL
, 0, 1);
371 EXPORT_SYMBOL(refresh_vm_stats
);
377 * zonelist = the list of zones passed to the allocator
378 * z = the zone from which the allocation occurred.
380 * Must be called with interrupts disabled.
382 void zone_statistics(struct zonelist
*zonelist
, struct zone
*z
)
384 if (z
->zone_pgdat
== zonelist
->zones
[0]->zone_pgdat
) {
385 __inc_zone_state(z
, NUMA_HIT
);
387 __inc_zone_state(z
, NUMA_MISS
);
388 __inc_zone_state(zonelist
->zones
[0], NUMA_FOREIGN
);
390 if (z
->node
== numa_node_id())
391 __inc_zone_state(z
, NUMA_LOCAL
);
393 __inc_zone_state(z
, NUMA_OTHER
);
397 #ifdef CONFIG_PROC_FS
399 #include <linux/seq_file.h>
401 static char * const migratetype_names
[MIGRATE_TYPES
] = {
408 static void *frag_start(struct seq_file
*m
, loff_t
*pos
)
412 for (pgdat
= first_online_pgdat();
414 pgdat
= next_online_pgdat(pgdat
))
420 static void *frag_next(struct seq_file
*m
, void *arg
, loff_t
*pos
)
422 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
425 return next_online_pgdat(pgdat
);
428 static void frag_stop(struct seq_file
*m
, void *arg
)
432 /* Walk all the zones in a node and print using a callback */
433 static void walk_zones_in_node(struct seq_file
*m
, pg_data_t
*pgdat
,
434 void (*print
)(struct seq_file
*m
, pg_data_t
*, struct zone
*))
437 struct zone
*node_zones
= pgdat
->node_zones
;
440 for (zone
= node_zones
; zone
- node_zones
< MAX_NR_ZONES
; ++zone
) {
441 if (!populated_zone(zone
))
444 spin_lock_irqsave(&zone
->lock
, flags
);
445 print(m
, pgdat
, zone
);
446 spin_unlock_irqrestore(&zone
->lock
, flags
);
450 static void frag_show_print(struct seq_file
*m
, pg_data_t
*pgdat
,
455 seq_printf(m
, "Node %d, zone %8s ", pgdat
->node_id
, zone
->name
);
456 for (order
= 0; order
< MAX_ORDER
; ++order
)
457 seq_printf(m
, "%6lu ", zone
->free_area
[order
].nr_free
);
462 * This walks the free areas for each zone.
464 static int frag_show(struct seq_file
*m
, void *arg
)
466 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
467 walk_zones_in_node(m
, pgdat
, frag_show_print
);
471 static void pagetypeinfo_showfree_print(struct seq_file
*m
,
472 pg_data_t
*pgdat
, struct zone
*zone
)
476 for (mtype
= 0; mtype
< MIGRATE_TYPES
; mtype
++) {
477 seq_printf(m
, "Node %4d, zone %8s, type %12s ",
480 migratetype_names
[mtype
]);
481 for (order
= 0; order
< MAX_ORDER
; ++order
) {
482 unsigned long freecount
= 0;
483 struct free_area
*area
;
484 struct list_head
*curr
;
486 area
= &(zone
->free_area
[order
]);
488 list_for_each(curr
, &area
->free_list
[mtype
])
490 seq_printf(m
, "%6lu ", freecount
);
496 /* Print out the free pages at each order for each migatetype */
497 static int pagetypeinfo_showfree(struct seq_file
*m
, void *arg
)
500 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
503 seq_printf(m
, "%-43s ", "Free pages count per migrate type at order");
504 for (order
= 0; order
< MAX_ORDER
; ++order
)
505 seq_printf(m
, "%6d ", order
);
508 walk_zones_in_node(m
, pgdat
, pagetypeinfo_showfree_print
);
513 static void pagetypeinfo_showblockcount_print(struct seq_file
*m
,
514 pg_data_t
*pgdat
, struct zone
*zone
)
518 unsigned long start_pfn
= zone
->zone_start_pfn
;
519 unsigned long end_pfn
= start_pfn
+ zone
->spanned_pages
;
520 unsigned long count
[MIGRATE_TYPES
] = { 0, };
522 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= pageblock_nr_pages
) {
528 page
= pfn_to_page(pfn
);
529 mtype
= get_pageblock_migratetype(page
);
535 seq_printf(m
, "Node %d, zone %8s ", pgdat
->node_id
, zone
->name
);
536 for (mtype
= 0; mtype
< MIGRATE_TYPES
; mtype
++)
537 seq_printf(m
, "%12lu ", count
[mtype
]);
541 /* Print out the free pages at each order for each migratetype */
542 static int pagetypeinfo_showblockcount(struct seq_file
*m
, void *arg
)
545 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
547 seq_printf(m
, "\n%-23s", "Number of blocks type ");
548 for (mtype
= 0; mtype
< MIGRATE_TYPES
; mtype
++)
549 seq_printf(m
, "%12s ", migratetype_names
[mtype
]);
551 walk_zones_in_node(m
, pgdat
, pagetypeinfo_showblockcount_print
);
557 * This prints out statistics in relation to grouping pages by mobility.
558 * It is expensive to collect so do not constantly read the file.
560 static int pagetypeinfo_show(struct seq_file
*m
, void *arg
)
562 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
564 seq_printf(m
, "Page block order: %d\n", pageblock_order
);
565 seq_printf(m
, "Pages per block: %lu\n", pageblock_nr_pages
);
567 pagetypeinfo_showfree(m
, pgdat
);
568 pagetypeinfo_showblockcount(m
, pgdat
);
573 const struct seq_operations fragmentation_op
= {
580 const struct seq_operations pagetypeinfo_op
= {
584 .show
= pagetypeinfo_show
,
587 #ifdef CONFIG_ZONE_DMA
588 #define TEXT_FOR_DMA(xx) xx "_dma",
590 #define TEXT_FOR_DMA(xx)
593 #ifdef CONFIG_ZONE_DMA32
594 #define TEXT_FOR_DMA32(xx) xx "_dma32",
596 #define TEXT_FOR_DMA32(xx)
599 #ifdef CONFIG_HIGHMEM
600 #define TEXT_FOR_HIGHMEM(xx) xx "_high",
602 #define TEXT_FOR_HIGHMEM(xx)
605 #define TEXTS_FOR_ZONES(xx) TEXT_FOR_DMA(xx) TEXT_FOR_DMA32(xx) xx "_normal", \
606 TEXT_FOR_HIGHMEM(xx) xx "_movable",
608 static const char * const vmstat_text
[] = {
609 /* Zoned VM counters */
618 "nr_slab_reclaimable",
619 "nr_slab_unreclaimable",
620 "nr_page_table_pages",
634 #ifdef CONFIG_VM_EVENT_COUNTERS
640 TEXTS_FOR_ZONES("pgalloc")
649 TEXTS_FOR_ZONES("pgrefill")
650 TEXTS_FOR_ZONES("pgsteal")
651 TEXTS_FOR_ZONES("pgscan_kswapd")
652 TEXTS_FOR_ZONES("pgscan_direct")
665 static void zoneinfo_show_print(struct seq_file
*m
, pg_data_t
*pgdat
,
669 seq_printf(m
, "Node %d, zone %8s", pgdat
->node_id
, zone
->name
);
675 "\n scanned %lu (a: %lu i: %lu)"
678 zone_page_state(zone
, NR_FREE_PAGES
),
683 zone
->nr_scan_active
, zone
->nr_scan_inactive
,
685 zone
->present_pages
);
687 for (i
= 0; i
< NR_VM_ZONE_STAT_ITEMS
; i
++)
688 seq_printf(m
, "\n %-12s %lu", vmstat_text
[i
],
689 zone_page_state(zone
, i
));
692 "\n protection: (%lu",
693 zone
->lowmem_reserve
[0]);
694 for (i
= 1; i
< ARRAY_SIZE(zone
->lowmem_reserve
); i
++)
695 seq_printf(m
, ", %lu", zone
->lowmem_reserve
[i
]);
699 for_each_online_cpu(i
) {
700 struct per_cpu_pageset
*pageset
;
703 pageset
= zone_pcp(zone
, i
);
704 for (j
= 0; j
< ARRAY_SIZE(pageset
->pcp
); j
++) {
711 pageset
->pcp
[j
].count
,
712 pageset
->pcp
[j
].high
,
713 pageset
->pcp
[j
].batch
);
716 seq_printf(m
, "\n vm stats threshold: %d",
717 pageset
->stat_threshold
);
721 "\n all_unreclaimable: %u"
722 "\n prev_priority: %i"
724 zone
->all_unreclaimable
,
726 zone
->zone_start_pfn
);
731 * Output information about zones in @pgdat.
733 static int zoneinfo_show(struct seq_file
*m
, void *arg
)
735 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
736 walk_zones_in_node(m
, pgdat
, zoneinfo_show_print
);
740 const struct seq_operations zoneinfo_op
= {
741 .start
= frag_start
, /* iterate over all zones. The same as in
745 .show
= zoneinfo_show
,
748 static void *vmstat_start(struct seq_file
*m
, loff_t
*pos
)
751 #ifdef CONFIG_VM_EVENT_COUNTERS
756 if (*pos
>= ARRAY_SIZE(vmstat_text
))
759 #ifdef CONFIG_VM_EVENT_COUNTERS
760 v
= kmalloc(NR_VM_ZONE_STAT_ITEMS
* sizeof(unsigned long)
761 + sizeof(struct vm_event_state
), GFP_KERNEL
);
763 v
= kmalloc(NR_VM_ZONE_STAT_ITEMS
* sizeof(unsigned long),
768 return ERR_PTR(-ENOMEM
);
769 for (i
= 0; i
< NR_VM_ZONE_STAT_ITEMS
; i
++)
770 v
[i
] = global_page_state(i
);
771 #ifdef CONFIG_VM_EVENT_COUNTERS
772 e
= v
+ NR_VM_ZONE_STAT_ITEMS
;
774 e
[PGPGIN
] /= 2; /* sectors -> kbytes */
780 static void *vmstat_next(struct seq_file
*m
, void *arg
, loff_t
*pos
)
783 if (*pos
>= ARRAY_SIZE(vmstat_text
))
785 return (unsigned long *)m
->private + *pos
;
788 static int vmstat_show(struct seq_file
*m
, void *arg
)
790 unsigned long *l
= arg
;
791 unsigned long off
= l
- (unsigned long *)m
->private;
793 seq_printf(m
, "%s %lu\n", vmstat_text
[off
], *l
);
797 static void vmstat_stop(struct seq_file
*m
, void *arg
)
803 const struct seq_operations vmstat_op
= {
804 .start
= vmstat_start
,
810 #endif /* CONFIG_PROC_FS */
813 static DEFINE_PER_CPU(struct delayed_work
, vmstat_work
);
814 int sysctl_stat_interval __read_mostly
= HZ
;
816 static void vmstat_update(struct work_struct
*w
)
818 refresh_cpu_vm_stats(smp_processor_id());
819 schedule_delayed_work(&__get_cpu_var(vmstat_work
),
820 sysctl_stat_interval
);
823 static void __devinit
start_cpu_timer(int cpu
)
825 struct delayed_work
*vmstat_work
= &per_cpu(vmstat_work
, cpu
);
827 INIT_DELAYED_WORK_DEFERRABLE(vmstat_work
, vmstat_update
);
828 schedule_delayed_work_on(cpu
, vmstat_work
, HZ
+ cpu
);
832 * Use the cpu notifier to insure that the thresholds are recalculated
835 static int __cpuinit
vmstat_cpuup_callback(struct notifier_block
*nfb
,
836 unsigned long action
,
839 long cpu
= (long)hcpu
;
843 case CPU_ONLINE_FROZEN
:
844 start_cpu_timer(cpu
);
846 case CPU_DOWN_PREPARE
:
847 case CPU_DOWN_PREPARE_FROZEN
:
848 cancel_rearming_delayed_work(&per_cpu(vmstat_work
, cpu
));
849 per_cpu(vmstat_work
, cpu
).work
.func
= NULL
;
851 case CPU_DOWN_FAILED
:
852 case CPU_DOWN_FAILED_FROZEN
:
853 start_cpu_timer(cpu
);
856 case CPU_DEAD_FROZEN
:
857 refresh_zone_stat_thresholds();
865 static struct notifier_block __cpuinitdata vmstat_notifier
=
866 { &vmstat_cpuup_callback
, NULL
, 0 };
868 int __init
setup_vmstat(void)
872 refresh_zone_stat_thresholds();
873 register_cpu_notifier(&vmstat_notifier
);
875 for_each_online_cpu(cpu
)
876 start_cpu_timer(cpu
);
879 module_init(setup_vmstat
)