Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | #ifndef __LINUX_PERCPU_H |
2 | #define __LINUX_PERCPU_H | |
7ff6f082 | 3 | |
1da177e4 LT |
4 | #include <linux/spinlock.h> /* For preempt_disable() */ |
5 | #include <linux/slab.h> /* For kmalloc() */ | |
6 | #include <linux/smp.h> | |
7 | #include <linux/string.h> /* For memset() */ | |
7ff6f082 MP |
8 | #include <linux/cpumask.h> |
9 | ||
1da177e4 LT |
10 | #include <asm/percpu.h> |
11 | ||
12 | /* Enough to cover all DEFINE_PER_CPUs in kernel, including modules. */ | |
13 | #ifndef PERCPU_ENOUGH_ROOM | |
b00742d3 JF |
14 | #ifdef CONFIG_MODULES |
15 | #define PERCPU_MODULE_RESERVE 8192 | |
16 | #else | |
17 | #define PERCPU_MODULE_RESERVE 0 | |
1da177e4 LT |
18 | #endif |
19 | ||
b00742d3 JF |
20 | #define PERCPU_ENOUGH_ROOM \ |
21 | (__per_cpu_end - __per_cpu_start + PERCPU_MODULE_RESERVE) | |
22 | #endif /* PERCPU_ENOUGH_ROOM */ | |
23 | ||
632bbfee JB |
24 | /* |
25 | * Must be an lvalue. Since @var must be a simple identifier, | |
26 | * we force a syntax error here if it isn't. | |
27 | */ | |
28 | #define get_cpu_var(var) (*({ \ | |
a666ecfb | 29 | extern int simple_identifier_##var(void); \ |
632bbfee JB |
30 | preempt_disable(); \ |
31 | &__get_cpu_var(var); })) | |
1da177e4 LT |
32 | #define put_cpu_var(var) preempt_enable() |
33 | ||
34 | #ifdef CONFIG_SMP | |
35 | ||
36 | struct percpu_data { | |
37 | void *ptrs[NR_CPUS]; | |
1da177e4 LT |
38 | }; |
39 | ||
7ff6f082 | 40 | #define __percpu_disguise(pdata) (struct percpu_data *)~(unsigned long)(pdata) |
1da177e4 | 41 | /* |
7ff6f082 MP |
42 | * Use this to get to a cpu's version of the per-cpu object dynamically |
43 | * allocated. Non-atomic access to the current CPU's version should | |
1da177e4 LT |
44 | * probably be combined with get_cpu()/put_cpu(). |
45 | */ | |
7ff6f082 MP |
46 | #define percpu_ptr(ptr, cpu) \ |
47 | ({ \ | |
48 | struct percpu_data *__p = __percpu_disguise(ptr); \ | |
49 | (__typeof__(ptr))__p->ptrs[(cpu)]; \ | |
1da177e4 LT |
50 | }) |
51 | ||
7ff6f082 MP |
52 | extern void *percpu_populate(void *__pdata, size_t size, gfp_t gfp, int cpu); |
53 | extern void percpu_depopulate(void *__pdata, int cpu); | |
54 | extern int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp, | |
55 | cpumask_t *mask); | |
56 | extern void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask); | |
57 | extern void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask); | |
58 | extern void percpu_free(void *__pdata); | |
1da177e4 LT |
59 | |
60 | #else /* CONFIG_SMP */ | |
61 | ||
7ff6f082 MP |
62 | #define percpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); }) |
63 | ||
64 | static inline void percpu_depopulate(void *__pdata, int cpu) | |
65 | { | |
66 | } | |
67 | ||
68 | static inline void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask) | |
69 | { | |
70 | } | |
1da177e4 | 71 | |
7ff6f082 MP |
72 | static inline void *percpu_populate(void *__pdata, size_t size, gfp_t gfp, |
73 | int cpu) | |
1da177e4 | 74 | { |
7ff6f082 | 75 | return percpu_ptr(__pdata, cpu); |
1da177e4 | 76 | } |
7ff6f082 MP |
77 | |
78 | static inline int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp, | |
79 | cpumask_t *mask) | |
80 | { | |
81 | return 0; | |
82 | } | |
83 | ||
0891a8d7 | 84 | static __always_inline void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask) |
7ff6f082 MP |
85 | { |
86 | return kzalloc(size, gfp); | |
87 | } | |
88 | ||
89 | static inline void percpu_free(void *__pdata) | |
90 | { | |
91 | kfree(__pdata); | |
1da177e4 LT |
92 | } |
93 | ||
94 | #endif /* CONFIG_SMP */ | |
95 | ||
7ff6f082 MP |
96 | #define percpu_populate_mask(__pdata, size, gfp, mask) \ |
97 | __percpu_populate_mask((__pdata), (size), (gfp), &(mask)) | |
98 | #define percpu_depopulate_mask(__pdata, mask) \ | |
99 | __percpu_depopulate_mask((__pdata), &(mask)) | |
100 | #define percpu_alloc_mask(size, gfp, mask) \ | |
101 | __percpu_alloc_mask((size), (gfp), &(mask)) | |
102 | ||
103 | #define percpu_alloc(size, gfp) percpu_alloc_mask((size), (gfp), cpu_online_map) | |
104 | ||
105 | /* (legacy) interface for use without CPU hotplug handling */ | |
106 | ||
107 | #define __alloc_percpu(size) percpu_alloc_mask((size), GFP_KERNEL, \ | |
108 | cpu_possible_map) | |
109 | #define alloc_percpu(type) (type *)__alloc_percpu(sizeof(type)) | |
110 | #define free_percpu(ptr) percpu_free((ptr)) | |
111 | #define per_cpu_ptr(ptr, cpu) percpu_ptr((ptr), (cpu)) | |
1da177e4 LT |
112 | |
113 | #endif /* __LINUX_PERCPU_H */ |