Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | #ifndef __LINUX_PERCPU_H |
2 | #define __LINUX_PERCPU_H | |
7ff6f082 | 3 | |
0a3021f4 | 4 | #include <linux/preempt.h> |
1da177e4 LT |
5 | #include <linux/slab.h> /* For kmalloc() */ |
6 | #include <linux/smp.h> | |
7ff6f082 MP |
7 | #include <linux/cpumask.h> |
8 | ||
1da177e4 LT |
9 | #include <asm/percpu.h> |
10 | ||
5280e004 | 11 | #ifdef CONFIG_SMP |
12 | #define DEFINE_PER_CPU(type, name) \ | |
13 | __attribute__((__section__(".data.percpu"))) \ | |
14 | PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name | |
15 | ||
44c81433 ED |
16 | #ifdef MODULE |
17 | #define SHARED_ALIGNED_SECTION ".data.percpu" | |
18 | #else | |
19 | #define SHARED_ALIGNED_SECTION ".data.percpu.shared_aligned" | |
20 | #endif | |
21 | ||
5280e004 | 22 | #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ |
44c81433 | 23 | __attribute__((__section__(SHARED_ALIGNED_SECTION))) \ |
5280e004 | 24 | PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name \ |
25 | ____cacheline_aligned_in_smp | |
63cc8c75 ED |
26 | |
27 | #define DEFINE_PER_CPU_PAGE_ALIGNED(type, name) \ | |
28 | __attribute__((__section__(".data.percpu.page_aligned"))) \ | |
29 | PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name | |
5280e004 | 30 | #else |
31 | #define DEFINE_PER_CPU(type, name) \ | |
32 | PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name | |
33 | ||
34 | #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \ | |
35 | DEFINE_PER_CPU(type, name) | |
63cc8c75 ED |
36 | |
37 | #define DEFINE_PER_CPU_PAGE_ALIGNED(type, name) \ | |
38 | DEFINE_PER_CPU(type, name) | |
5280e004 | 39 | #endif |
40 | ||
41 | #define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var) | |
42 | #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var) | |
43 | ||
1da177e4 LT |
44 | /* Enough to cover all DEFINE_PER_CPUs in kernel, including modules. */ |
45 | #ifndef PERCPU_ENOUGH_ROOM | |
b00742d3 JF |
46 | #ifdef CONFIG_MODULES |
47 | #define PERCPU_MODULE_RESERVE 8192 | |
48 | #else | |
49 | #define PERCPU_MODULE_RESERVE 0 | |
1da177e4 LT |
50 | #endif |
51 | ||
b00742d3 JF |
52 | #define PERCPU_ENOUGH_ROOM \ |
53 | (__per_cpu_end - __per_cpu_start + PERCPU_MODULE_RESERVE) | |
54 | #endif /* PERCPU_ENOUGH_ROOM */ | |
55 | ||
632bbfee JB |
56 | /* |
57 | * Must be an lvalue. Since @var must be a simple identifier, | |
58 | * we force a syntax error here if it isn't. | |
59 | */ | |
60 | #define get_cpu_var(var) (*({ \ | |
a666ecfb | 61 | extern int simple_identifier_##var(void); \ |
632bbfee JB |
62 | preempt_disable(); \ |
63 | &__get_cpu_var(var); })) | |
1da177e4 LT |
64 | #define put_cpu_var(var) preempt_enable() |
65 | ||
66 | #ifdef CONFIG_SMP | |
67 | ||
68 | struct percpu_data { | |
b3242151 | 69 | void *ptrs[1]; |
1da177e4 LT |
70 | }; |
71 | ||
7ff6f082 | 72 | #define __percpu_disguise(pdata) (struct percpu_data *)~(unsigned long)(pdata) |
1da177e4 | 73 | /* |
7ff6f082 MP |
74 | * Use this to get to a cpu's version of the per-cpu object dynamically |
75 | * allocated. Non-atomic access to the current CPU's version should | |
1da177e4 LT |
76 | * probably be combined with get_cpu()/put_cpu(). |
77 | */ | |
7ff6f082 MP |
78 | #define percpu_ptr(ptr, cpu) \ |
79 | ({ \ | |
80 | struct percpu_data *__p = __percpu_disguise(ptr); \ | |
81 | (__typeof__(ptr))__p->ptrs[(cpu)]; \ | |
1da177e4 LT |
82 | }) |
83 | ||
7ff6f082 MP |
84 | extern void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask); |
85 | extern void percpu_free(void *__pdata); | |
1da177e4 LT |
86 | |
87 | #else /* CONFIG_SMP */ | |
88 | ||
7ff6f082 MP |
89 | #define percpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); }) |
90 | ||
0891a8d7 | 91 | static __always_inline void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask) |
7ff6f082 MP |
92 | { |
93 | return kzalloc(size, gfp); | |
94 | } | |
95 | ||
96 | static inline void percpu_free(void *__pdata) | |
97 | { | |
98 | kfree(__pdata); | |
1da177e4 LT |
99 | } |
100 | ||
101 | #endif /* CONFIG_SMP */ | |
102 | ||
7ff6f082 MP |
103 | #define percpu_alloc_mask(size, gfp, mask) \ |
104 | __percpu_alloc_mask((size), (gfp), &(mask)) | |
105 | ||
106 | #define percpu_alloc(size, gfp) percpu_alloc_mask((size), (gfp), cpu_online_map) | |
107 | ||
108 | /* (legacy) interface for use without CPU hotplug handling */ | |
109 | ||
110 | #define __alloc_percpu(size) percpu_alloc_mask((size), GFP_KERNEL, \ | |
111 | cpu_possible_map) | |
112 | #define alloc_percpu(type) (type *)__alloc_percpu(sizeof(type)) | |
113 | #define free_percpu(ptr) percpu_free((ptr)) | |
114 | #define per_cpu_ptr(ptr, cpu) percpu_ptr((ptr), (cpu)) | |
1da177e4 LT |
115 | |
116 | #endif /* __LINUX_PERCPU_H */ |