2 * Fast batching percpu counters.
5 #include <linux/percpu_counter.h>
6 #include <linux/notifier.h>
7 #include <linux/mutex.h>
8 #include <linux/init.h>
10 #include <linux/module.h>
12 #ifdef CONFIG_HOTPLUG_CPU
13 static LIST_HEAD(percpu_counters
);
14 static DEFINE_MUTEX(percpu_counters_lock
);
17 void percpu_counter_set(struct percpu_counter
*fbc
, s64 amount
)
21 spin_lock(&fbc
->lock
);
22 for_each_possible_cpu(cpu
) {
23 s32
*pcount
= per_cpu_ptr(fbc
->counters
, cpu
);
27 spin_unlock(&fbc
->lock
);
29 EXPORT_SYMBOL(percpu_counter_set
);
31 void __percpu_counter_add(struct percpu_counter
*fbc
, s64 amount
, s32 batch
)
37 pcount
= per_cpu_ptr(fbc
->counters
, cpu
);
38 count
= *pcount
+ amount
;
39 if (count
>= batch
|| count
<= -batch
) {
40 spin_lock(&fbc
->lock
);
43 spin_unlock(&fbc
->lock
);
49 EXPORT_SYMBOL(__percpu_counter_add
);
52 * Add up all the per-cpu counts, return the result. This is a more accurate
53 * but much slower version of percpu_counter_read_positive()
55 s64
__percpu_counter_sum(struct percpu_counter
*fbc
)
60 spin_lock(&fbc
->lock
);
62 for_each_online_cpu(cpu
) {
63 s32
*pcount
= per_cpu_ptr(fbc
->counters
, cpu
);
69 spin_unlock(&fbc
->lock
);
72 EXPORT_SYMBOL(__percpu_counter_sum
);
74 int __percpu_counter_init(struct percpu_counter
*fbc
, s64 amount
,
75 struct lock_class_key
*key
)
77 spin_lock_init(&fbc
->lock
);
78 lockdep_set_class(&fbc
->lock
, key
);
80 fbc
->counters
= alloc_percpu(s32
);
83 #ifdef CONFIG_HOTPLUG_CPU
84 mutex_lock(&percpu_counters_lock
);
85 list_add(&fbc
->list
, &percpu_counters
);
86 mutex_unlock(&percpu_counters_lock
);
90 EXPORT_SYMBOL(__percpu_counter_init
);
92 void percpu_counter_destroy(struct percpu_counter
*fbc
)
97 free_percpu(fbc
->counters
);
99 #ifdef CONFIG_HOTPLUG_CPU
100 mutex_lock(&percpu_counters_lock
);
101 list_del(&fbc
->list
);
102 mutex_unlock(&percpu_counters_lock
);
105 EXPORT_SYMBOL(percpu_counter_destroy
);
107 #ifdef CONFIG_HOTPLUG_CPU
108 static int __cpuinit
percpu_counter_hotcpu_callback(struct notifier_block
*nb
,
109 unsigned long action
, void *hcpu
)
112 struct percpu_counter
*fbc
;
114 if (action
!= CPU_DEAD
)
117 cpu
= (unsigned long)hcpu
;
118 mutex_lock(&percpu_counters_lock
);
119 list_for_each_entry(fbc
, &percpu_counters
, list
) {
123 spin_lock_irqsave(&fbc
->lock
, flags
);
124 pcount
= per_cpu_ptr(fbc
->counters
, cpu
);
125 fbc
->count
+= *pcount
;
127 spin_unlock_irqrestore(&fbc
->lock
, flags
);
129 mutex_unlock(&percpu_counters_lock
);
133 static int __init
percpu_counter_startup(void)
135 hotcpu_notifier(percpu_counter_hotcpu_callback
, 0);
138 module_init(percpu_counter_startup
);