Commit | Line | Data |
---|---|---|
d00bcc98 CL |
1 | /* |
2 | * linux/mm/allocpercpu.c | |
3 | * | |
cde53535 | 4 | * Separated from slab.c August 11, 2006 Christoph Lameter |
d00bcc98 CL |
5 | */ |
6 | #include <linux/mm.h> | |
7 | #include <linux/module.h> | |
e74e3962 TH |
8 | #include <linux/bootmem.h> |
9 | #include <asm/sections.h> | |
d00bcc98 | 10 | |
be852795 ED |
11 | #ifndef cache_line_size |
12 | #define cache_line_size() L1_CACHE_BYTES | |
13 | #endif | |
14 | ||
d00bcc98 CL |
15 | /** |
16 | * percpu_depopulate - depopulate per-cpu data for given cpu | |
17 | * @__pdata: per-cpu data to depopulate | |
18 | * @cpu: depopulate per-cpu data for this cpu | |
19 | * | |
20 | * Depopulating per-cpu data for a cpu going offline would be a typical | |
21 | * use case. You need to register a cpu hotplug handler for that purpose. | |
22 | */ | |
9d8fddfb | 23 | static void percpu_depopulate(void *__pdata, int cpu) |
d00bcc98 CL |
24 | { |
25 | struct percpu_data *pdata = __percpu_disguise(__pdata); | |
a1205868 AS |
26 | |
27 | kfree(pdata->ptrs[cpu]); | |
28 | pdata->ptrs[cpu] = NULL; | |
d00bcc98 | 29 | } |
d00bcc98 CL |
30 | |
31 | /** | |
32 | * percpu_depopulate_mask - depopulate per-cpu data for some cpu's | |
33 | * @__pdata: per-cpu data to depopulate | |
34 | * @mask: depopulate per-cpu data for cpu's selected through mask bits | |
35 | */ | |
5d6700ea | 36 | static void __percpu_depopulate_mask(void *__pdata, const cpumask_t *mask) |
d00bcc98 CL |
37 | { |
38 | int cpu; | |
6d6a4360 | 39 | for_each_cpu_mask_nr(cpu, *mask) |
d00bcc98 CL |
40 | percpu_depopulate(__pdata, cpu); |
41 | } | |
9d8fddfb AB |
42 | |
43 | #define percpu_depopulate_mask(__pdata, mask) \ | |
44 | __percpu_depopulate_mask((__pdata), &(mask)) | |
d00bcc98 CL |
45 | |
46 | /** | |
47 | * percpu_populate - populate per-cpu data for given cpu | |
48 | * @__pdata: per-cpu data to populate further | |
49 | * @size: size of per-cpu object | |
50 | * @gfp: may sleep or not etc. | |
51 | * @cpu: populate per-data for this cpu | |
52 | * | |
53 | * Populating per-cpu data for a cpu coming online would be a typical | |
54 | * use case. You need to register a cpu hotplug handler for that purpose. | |
55 | * Per-cpu object is populated with zeroed buffer. | |
56 | */ | |
9d8fddfb | 57 | static void *percpu_populate(void *__pdata, size_t size, gfp_t gfp, int cpu) |
d00bcc98 CL |
58 | { |
59 | struct percpu_data *pdata = __percpu_disguise(__pdata); | |
60 | int node = cpu_to_node(cpu); | |
61 | ||
be852795 ED |
62 | /* |
63 | * We should make sure each CPU gets private memory. | |
64 | */ | |
65 | size = roundup(size, cache_line_size()); | |
66 | ||
d00bcc98 | 67 | BUG_ON(pdata->ptrs[cpu]); |
94f6030c CL |
68 | if (node_online(node)) |
69 | pdata->ptrs[cpu] = kmalloc_node(size, gfp|__GFP_ZERO, node); | |
70 | else | |
d00bcc98 CL |
71 | pdata->ptrs[cpu] = kzalloc(size, gfp); |
72 | return pdata->ptrs[cpu]; | |
73 | } | |
d00bcc98 CL |
74 | |
75 | /** | |
76 | * percpu_populate_mask - populate per-cpu data for more cpu's | |
77 | * @__pdata: per-cpu data to populate further | |
78 | * @size: size of per-cpu object | |
79 | * @gfp: may sleep or not etc. | |
80 | * @mask: populate per-cpu data for cpu's selected through mask bits | |
81 | * | |
82 | * Per-cpu objects are populated with zeroed buffers. | |
83 | */ | |
9d8fddfb AB |
84 | static int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp, |
85 | cpumask_t *mask) | |
d00bcc98 | 86 | { |
d366f8cb | 87 | cpumask_t populated; |
d00bcc98 CL |
88 | int cpu; |
89 | ||
d366f8cb | 90 | cpus_clear(populated); |
6d6a4360 | 91 | for_each_cpu_mask_nr(cpu, *mask) |
d00bcc98 CL |
92 | if (unlikely(!percpu_populate(__pdata, size, gfp, cpu))) { |
93 | __percpu_depopulate_mask(__pdata, &populated); | |
94 | return -ENOMEM; | |
95 | } else | |
96 | cpu_set(cpu, populated); | |
97 | return 0; | |
98 | } | |
9d8fddfb AB |
99 | |
100 | #define percpu_populate_mask(__pdata, size, gfp, mask) \ | |
101 | __percpu_populate_mask((__pdata), (size), (gfp), &(mask)) | |
d00bcc98 CL |
102 | |
103 | /** | |
f2a8205c | 104 | * alloc_percpu - initial setup of per-cpu data |
d00bcc98 | 105 | * @size: size of per-cpu object |
f2a8205c | 106 | * @align: alignment |
d00bcc98 | 107 | * |
f2a8205c TH |
108 | * Allocate dynamic percpu area. Percpu objects are populated with |
109 | * zeroed buffers. | |
d00bcc98 | 110 | */ |
f2a8205c | 111 | void *__alloc_percpu(size_t size, size_t align) |
d00bcc98 | 112 | { |
be852795 ED |
113 | /* |
114 | * We allocate whole cache lines to avoid false sharing | |
115 | */ | |
116 | size_t sz = roundup(nr_cpu_ids * sizeof(void *), cache_line_size()); | |
f2a8205c | 117 | void *pdata = kzalloc(sz, GFP_KERNEL); |
d00bcc98 CL |
118 | void *__pdata = __percpu_disguise(pdata); |
119 | ||
f2a8205c TH |
120 | /* |
121 | * Can't easily make larger alignment work with kmalloc. WARN | |
122 | * on it. Larger alignment should only be used for module | |
123 | * percpu sections on SMP for which this path isn't used. | |
124 | */ | |
60db5642 | 125 | WARN_ON_ONCE(align > SMP_CACHE_BYTES); |
f2a8205c | 126 | |
d00bcc98 CL |
127 | if (unlikely(!pdata)) |
128 | return NULL; | |
f2a8205c TH |
129 | if (likely(!__percpu_populate_mask(__pdata, size, GFP_KERNEL, |
130 | &cpu_possible_map))) | |
d00bcc98 CL |
131 | return __pdata; |
132 | kfree(pdata); | |
133 | return NULL; | |
134 | } | |
f2a8205c | 135 | EXPORT_SYMBOL_GPL(__alloc_percpu); |
d00bcc98 CL |
136 | |
137 | /** | |
f2a8205c | 138 | * free_percpu - final cleanup of per-cpu data |
d00bcc98 CL |
139 | * @__pdata: object to clean up |
140 | * | |
141 | * We simply clean up any per-cpu object left. No need for the client to | |
142 | * track and specify through a bis mask which per-cpu objects are to free. | |
143 | */ | |
f2a8205c | 144 | void free_percpu(void *__pdata) |
d00bcc98 | 145 | { |
a1205868 AS |
146 | if (unlikely(!__pdata)) |
147 | return; | |
aa85ea5b | 148 | __percpu_depopulate_mask(__pdata, cpu_possible_mask); |
d00bcc98 CL |
149 | kfree(__percpu_disguise(__pdata)); |
150 | } | |
f2a8205c | 151 | EXPORT_SYMBOL_GPL(free_percpu); |
e74e3962 TH |
152 | |
153 | /* | |
154 | * Generic percpu area setup. | |
155 | */ | |
156 | #ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA | |
157 | unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; | |
158 | ||
159 | EXPORT_SYMBOL(__per_cpu_offset); | |
160 | ||
161 | void __init setup_per_cpu_areas(void) | |
162 | { | |
163 | unsigned long size, i; | |
164 | char *ptr; | |
165 | unsigned long nr_possible_cpus = num_possible_cpus(); | |
166 | ||
167 | /* Copy section for each CPU (we discard the original) */ | |
168 | size = ALIGN(PERCPU_ENOUGH_ROOM, PAGE_SIZE); | |
169 | ptr = alloc_bootmem_pages(size * nr_possible_cpus); | |
170 | ||
171 | for_each_possible_cpu(i) { | |
172 | __per_cpu_offset[i] = ptr - __per_cpu_start; | |
173 | memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start); | |
174 | ptr += size; | |
175 | } | |
176 | } | |
177 | #endif /* CONFIG_HAVE_SETUP_PER_CPU_AREA */ |