2 * Fast batching percpu counters.
5 #include <linux/percpu_counter.h>
6 #include <linux/notifier.h>
7 #include <linux/mutex.h>
8 #include <linux/init.h>
10 #include <linux/module.h>
12 #ifdef CONFIG_HOTPLUG_CPU
13 static LIST_HEAD(percpu_counters
);
14 static DEFINE_MUTEX(percpu_counters_lock
);
17 void percpu_counter_set(struct percpu_counter
*fbc
, s64 amount
)
21 spin_lock(&fbc
->lock
);
22 for_each_possible_cpu(cpu
) {
23 s32
*pcount
= per_cpu_ptr(fbc
->counters
, cpu
);
27 spin_unlock(&fbc
->lock
);
29 EXPORT_SYMBOL(percpu_counter_set
);
31 void __percpu_counter_add(struct percpu_counter
*fbc
, s64 amount
, s32 batch
)
37 pcount
= per_cpu_ptr(fbc
->counters
, cpu
);
38 count
= *pcount
+ amount
;
39 if (count
>= batch
|| count
<= -batch
) {
40 spin_lock(&fbc
->lock
);
43 spin_unlock(&fbc
->lock
);
49 EXPORT_SYMBOL(__percpu_counter_add
);
52 * Add up all the per-cpu counts, return the result. This is a more accurate
53 * but much slower version of percpu_counter_read_positive()
55 s64
__percpu_counter_sum(struct percpu_counter
*fbc
)
60 spin_lock(&fbc
->lock
);
62 for_each_online_cpu(cpu
) {
63 s32
*pcount
= per_cpu_ptr(fbc
->counters
, cpu
);
66 spin_unlock(&fbc
->lock
);
69 EXPORT_SYMBOL(__percpu_counter_sum
);
71 int __percpu_counter_init(struct percpu_counter
*fbc
, s64 amount
,
72 struct lock_class_key
*key
)
74 spin_lock_init(&fbc
->lock
);
75 lockdep_set_class(&fbc
->lock
, key
);
77 fbc
->counters
= alloc_percpu(s32
);
80 #ifdef CONFIG_HOTPLUG_CPU
81 mutex_lock(&percpu_counters_lock
);
82 list_add(&fbc
->list
, &percpu_counters
);
83 mutex_unlock(&percpu_counters_lock
);
87 EXPORT_SYMBOL(__percpu_counter_init
);
89 void percpu_counter_destroy(struct percpu_counter
*fbc
)
94 #ifdef CONFIG_HOTPLUG_CPU
95 mutex_lock(&percpu_counters_lock
);
97 mutex_unlock(&percpu_counters_lock
);
99 free_percpu(fbc
->counters
);
100 fbc
->counters
= NULL
;
102 EXPORT_SYMBOL(percpu_counter_destroy
);
104 #ifdef CONFIG_HOTPLUG_CPU
105 static int __cpuinit
percpu_counter_hotcpu_callback(struct notifier_block
*nb
,
106 unsigned long action
, void *hcpu
)
109 struct percpu_counter
*fbc
;
111 if (action
!= CPU_DEAD
)
114 cpu
= (unsigned long)hcpu
;
115 mutex_lock(&percpu_counters_lock
);
116 list_for_each_entry(fbc
, &percpu_counters
, list
) {
120 spin_lock_irqsave(&fbc
->lock
, flags
);
121 pcount
= per_cpu_ptr(fbc
->counters
, cpu
);
122 fbc
->count
+= *pcount
;
124 spin_unlock_irqrestore(&fbc
->lock
, flags
);
126 mutex_unlock(&percpu_counters_lock
);
130 static int __init
percpu_counter_startup(void)
132 hotcpu_notifier(percpu_counter_hotcpu_callback
, 0);
135 module_init(percpu_counter_startup
);