Commit | Line | Data |
---|---|---|
5a0e3ad6 | 1 | #include <linux/slab.h> |
ccb46000 AM |
2 | #include <linux/kernel.h> |
3 | #include <linux/bitops.h> | |
4 | #include <linux/cpumask.h> | |
8bc3bcc9 | 5 | #include <linux/export.h> |
2d3854a3 | 6 | #include <linux/bootmem.h> |
ccb46000 AM |
7 | |
8 | int __first_cpu(const cpumask_t *srcp) | |
9 | { | |
10 | return min_t(int, NR_CPUS, find_first_bit(srcp->bits, NR_CPUS)); | |
11 | } | |
12 | EXPORT_SYMBOL(__first_cpu); | |
13 | ||
3d18bd74 AM |
14 | int __next_cpu(int n, const cpumask_t *srcp) |
15 | { | |
16 | return min_t(int, NR_CPUS, find_next_bit(srcp->bits, NR_CPUS, n+1)); | |
17 | } | |
18 | EXPORT_SYMBOL(__next_cpu); | |
86302820 | 19 | |
41df0d61 MT |
20 | #if NR_CPUS > 64 |
21 | int __next_cpu_nr(int n, const cpumask_t *srcp) | |
22 | { | |
23 | return min_t(int, nr_cpu_ids, | |
24 | find_next_bit(srcp->bits, nr_cpu_ids, n+1)); | |
25 | } | |
26 | EXPORT_SYMBOL(__next_cpu_nr); | |
27 | #endif | |
28 | ||
2d3854a3 RR |
29 | /** |
30 | * cpumask_next_and - get the next cpu in *src1p & *src2p | |
31 | * @n: the cpu prior to the place to search (ie. return will be > @n) | |
32 | * @src1p: the first cpumask pointer | |
33 | * @src2p: the second cpumask pointer | |
34 | * | |
35 | * Returns >= nr_cpu_ids if no further cpus set in both. | |
36 | */ | |
37 | int cpumask_next_and(int n, const struct cpumask *src1p, | |
38 | const struct cpumask *src2p) | |
39 | { | |
534b483a SS |
40 | struct cpumask tmp; |
41 | ||
42 | if (cpumask_and(&tmp, src1p, src2p)) | |
43 | return cpumask_next(n, &tmp); | |
44 | return nr_cpu_ids; | |
2d3854a3 RR |
45 | } |
46 | EXPORT_SYMBOL(cpumask_next_and); | |
47 | ||
48 | /** | |
49 | * cpumask_any_but - return a "random" in a cpumask, but not this one. | |
50 | * @mask: the cpumask to search | |
51 | * @cpu: the cpu to ignore. | |
52 | * | |
53 | * Often used to find any cpu but smp_processor_id() in a mask. | |
54 | * Returns >= nr_cpu_ids if no cpus set. | |
55 | */ | |
56 | int cpumask_any_but(const struct cpumask *mask, unsigned int cpu) | |
57 | { | |
58 | unsigned int i; | |
59 | ||
984f2f37 | 60 | cpumask_check(cpu); |
2d3854a3 RR |
61 | for_each_cpu(i, mask) |
62 | if (i != cpu) | |
63 | break; | |
64 | return i; | |
65 | } | |
66 | ||
67 | /* These are not inline because of header tangles. */ | |
68 | #ifdef CONFIG_CPUMASK_OFFSTACK | |
ec26b805 MT |
69 | /** |
70 | * alloc_cpumask_var_node - allocate a struct cpumask on a given node | |
71 | * @mask: pointer to cpumask_var_t where the cpumask is returned | |
72 | * @flags: GFP_ flags | |
73 | * | |
74 | * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is | |
75 | * a nop returning a constant 1 (in <linux/cpumask.h>) | |
76 | * Returns TRUE if memory allocation succeeded, FALSE otherwise. | |
77 | * | |
78 | * In addition, mask will be NULL if this fails. Note that gcc is | |
79 | * usually smart enough to know that mask can never be NULL if | |
80 | * CONFIG_CPUMASK_OFFSTACK=n, so does code elimination in that case | |
81 | * too. | |
82 | */ | |
7b4967c5 | 83 | bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) |
2d3854a3 | 84 | { |
38c7fed2 YL |
85 | *mask = kmalloc_node(cpumask_size(), flags, node); |
86 | ||
2d3854a3 RR |
87 | #ifdef CONFIG_DEBUG_PER_CPU_MAPS |
88 | if (!*mask) { | |
89 | printk(KERN_ERR "=> alloc_cpumask_var: failed!\n"); | |
90 | dump_stack(); | |
91 | } | |
92 | #endif | |
2a530080 RR |
93 | /* FIXME: Bandaid to save us from old primitives which go to NR_CPUS. */ |
94 | if (*mask) { | |
4f032ac4 | 95 | unsigned char *ptr = (unsigned char *)cpumask_bits(*mask); |
2a530080 RR |
96 | unsigned int tail; |
97 | tail = BITS_TO_LONGS(NR_CPUS - nr_cpumask_bits) * sizeof(long); | |
4f032ac4 | 98 | memset(ptr + cpumask_size() - tail, 0, tail); |
2a530080 RR |
99 | } |
100 | ||
2d3854a3 RR |
101 | return *mask != NULL; |
102 | } | |
7b4967c5 MT |
103 | EXPORT_SYMBOL(alloc_cpumask_var_node); |
104 | ||
0281b5dc YL |
105 | bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) |
106 | { | |
107 | return alloc_cpumask_var_node(mask, flags | __GFP_ZERO, node); | |
108 | } | |
109 | EXPORT_SYMBOL(zalloc_cpumask_var_node); | |
110 | ||
ec26b805 MT |
111 | /** |
112 | * alloc_cpumask_var - allocate a struct cpumask | |
113 | * @mask: pointer to cpumask_var_t where the cpumask is returned | |
114 | * @flags: GFP_ flags | |
115 | * | |
116 | * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is | |
117 | * a nop returning a constant 1 (in <linux/cpumask.h>). | |
118 | * | |
119 | * See alloc_cpumask_var_node. | |
120 | */ | |
7b4967c5 MT |
121 | bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) |
122 | { | |
37e7b5f1 | 123 | return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE); |
7b4967c5 | 124 | } |
2d3854a3 RR |
125 | EXPORT_SYMBOL(alloc_cpumask_var); |
126 | ||
0281b5dc YL |
127 | bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) |
128 | { | |
129 | return alloc_cpumask_var(mask, flags | __GFP_ZERO); | |
130 | } | |
131 | EXPORT_SYMBOL(zalloc_cpumask_var); | |
132 | ||
ec26b805 MT |
133 | /** |
134 | * alloc_bootmem_cpumask_var - allocate a struct cpumask from the bootmem arena. | |
135 | * @mask: pointer to cpumask_var_t where the cpumask is returned | |
136 | * | |
137 | * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is | |
e9690a6e | 138 | * a nop (in <linux/cpumask.h>). |
ec26b805 MT |
139 | * Either returns an allocated (zero-filled) cpumask, or causes the |
140 | * system to panic. | |
141 | */ | |
2d3854a3 RR |
142 | void __init alloc_bootmem_cpumask_var(cpumask_var_t *mask) |
143 | { | |
c1529500 | 144 | *mask = memblock_virt_alloc(cpumask_size(), 0); |
2d3854a3 RR |
145 | } |
146 | ||
ec26b805 MT |
147 | /** |
148 | * free_cpumask_var - frees memory allocated for a struct cpumask. | |
149 | * @mask: cpumask to free | |
150 | * | |
151 | * This is safe on a NULL mask. | |
152 | */ | |
2d3854a3 RR |
153 | void free_cpumask_var(cpumask_var_t mask) |
154 | { | |
155 | kfree(mask); | |
156 | } | |
157 | EXPORT_SYMBOL(free_cpumask_var); | |
cd83e42c | 158 | |
ec26b805 MT |
159 | /** |
160 | * free_bootmem_cpumask_var - frees result of alloc_bootmem_cpumask_var | |
161 | * @mask: cpumask to free | |
162 | */ | |
984f2f37 | 163 | void __init free_bootmem_cpumask_var(cpumask_var_t mask) |
cd83e42c | 164 | { |
c1529500 | 165 | memblock_free_early(__pa(mask), cpumask_size()); |
cd83e42c | 166 | } |
2d3854a3 | 167 | #endif |
da91309e AV |
168 | |
169 | /** | |
170 | * cpumask_set_cpu_local_first - set i'th cpu with local numa cpu's first | |
171 | * | |
172 | * @i: index number | |
173 | * @numa_node: local numa_node | |
174 | * @dstp: cpumask with the relevant cpu bit set according to the policy | |
175 | * | |
176 | * This function sets the cpumask according to a numa aware policy. | |
177 | * cpumask could be used as an affinity hint for the IRQ related to a | |
178 | * queue. When the policy is to spread queues across cores - local cores | |
179 | * first. | |
180 | * | |
181 | * Returns 0 on success, -ENOMEM for no memory, and -EAGAIN when failed to set | |
182 | * the cpu bit and need to re-call the function. | |
183 | */ | |
184 | int cpumask_set_cpu_local_first(int i, int numa_node, cpumask_t *dstp) | |
185 | { | |
186 | cpumask_var_t mask; | |
187 | int cpu; | |
188 | int ret = 0; | |
189 | ||
190 | if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) | |
191 | return -ENOMEM; | |
192 | ||
193 | i %= num_online_cpus(); | |
194 | ||
143b5ba2 | 195 | if (numa_node == -1 || !cpumask_of_node(numa_node)) { |
da91309e AV |
196 | /* Use all online cpu's for non numa aware system */ |
197 | cpumask_copy(mask, cpu_online_mask); | |
198 | } else { | |
199 | int n; | |
200 | ||
201 | cpumask_and(mask, | |
202 | cpumask_of_node(numa_node), cpu_online_mask); | |
203 | ||
204 | n = cpumask_weight(mask); | |
205 | if (i >= n) { | |
206 | i -= n; | |
207 | ||
208 | /* If index > number of local cpu's, mask out local | |
209 | * cpu's | |
210 | */ | |
211 | cpumask_andnot(mask, cpu_online_mask, mask); | |
212 | } | |
213 | } | |
214 | ||
215 | for_each_cpu(cpu, mask) { | |
216 | if (--i < 0) | |
217 | goto out; | |
218 | } | |
219 | ||
220 | ret = -EAGAIN; | |
221 | ||
222 | out: | |
223 | free_cpumask_var(mask); | |
224 | ||
225 | if (!ret) | |
226 | cpumask_set_cpu(cpu, dstp); | |
227 | ||
228 | return ret; | |
229 | } | |
230 | EXPORT_SYMBOL(cpumask_set_cpu_local_first); |