projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
percpu: remove per_cpu__ prefix.
[deliverable/linux.git] / include / linux / percpu.h
CommitLineData
1da177e4
LT		 1#ifndef __LINUX_PERCPU_H
2#define __LINUX_PERCPU_H
7ff6f082 3
0a3021f4 4#include <linux/preempt.h>
1da177e4
LT		 5#include <linux/slab.h> /* For kmalloc() */
6#include <linux/smp.h>
7ff6f082 7#include <linux/cpumask.h>
6a242909 8#include <linux/pfn.h>
7ff6f082 9
1da177e4
LT		 10#include <asm/percpu.h>
11
6a242909 12/* enough to cover all DEFINE_PER_CPUs in modules */
b00742d3 13#ifdef CONFIG_MODULES
6a242909 14#define PERCPU_MODULE_RESERVE (8 << 10)
b00742d3 15#else
6a242909 16#define PERCPU_MODULE_RESERVE 0
1da177e4
LT		 17#endif
18
6a242909 19#ifndef PERCPU_ENOUGH_ROOM
b00742d3 20#define PERCPU_ENOUGH_ROOM \
6a242909
TH		 21 (ALIGN(__per_cpu_end - __per_cpu_start, SMP_CACHE_BYTES) + \
22 PERCPU_MODULE_RESERVE)
23#endif
b00742d3 24
632bbfee
JB		 25/*
26 * Must be an lvalue. Since @var must be a simple identifier,
27 * we force a syntax error here if it isn't.
28 */
29#define get_cpu_var(var) (*({ \
a666ecfb 30 extern int simple_identifier_##var(void); \
632bbfee
JB		 31 preempt_disable(); \
32 &__get_cpu_var(var); }))
1da177e4
LT		 33#define put_cpu_var(var) preempt_enable()
34
35#ifdef CONFIG_SMP
36
8d408b4b 37/* minimum unit size, also is the maximum supported allocation size */
6a242909 38#define PCPU_MIN_UNIT_SIZE PFN_ALIGN(64 << 10)
8d408b4b
TH		 39
40/*
41 * PERCPU_DYNAMIC_RESERVE indicates the amount of free area to piggy
6b19b0c2
TH		 42 * back on the first chunk for dynamic percpu allocation if arch is
43 * manually allocating and mapping it for faster access (as a part of
44 * large page mapping for example).
8d408b4b 45 *
6b19b0c2
TH		 46 * The following values give between one and two pages of free space
47 * after typical minimal boot (2-way SMP, single disk and NIC) with
48 * both defconfig and a distro config on x86_64 and 32. More
49 * intelligent way to determine this would be nice.
8d408b4b 50 */
6b19b0c2
TH		 51#if BITS_PER_LONG > 32
52#define PERCPU_DYNAMIC_RESERVE (20 << 10)
53#else
54#define PERCPU_DYNAMIC_RESERVE (12 << 10)
55#endif
8d408b4b 56
fbf59bc9 57extern void *pcpu_base_addr;
fb435d52 58extern const unsigned long *pcpu_unit_offsets;
1da177e4 59
fd1e8a1f
TH		 60struct pcpu_group_info {
61 int nr_units; /* aligned # of units */
62 unsigned long base_offset; /* base address offset */
63 unsigned int *cpu_map; /* unit->cpu map, empty
64 * entries contain NR_CPUS */
65};
66
67struct pcpu_alloc_info {
68 size_t static_size;
69 size_t reserved_size;
70 size_t dyn_size;
71 size_t unit_size;
72 size_t atom_size;
73 size_t alloc_size;
74 size_t __ai_size; /* internal, don't use */
75 int nr_groups; /* 0 if grouping unnecessary */
76 struct pcpu_group_info groups[];
77};
78
f58dc01b
TH		 79enum pcpu_fc {
80 PCPU_FC_AUTO,
81 PCPU_FC_EMBED,
82 PCPU_FC_PAGE,
f58dc01b
TH		 83
84 PCPU_FC_NR,
85};
86extern const char *pcpu_fc_names[PCPU_FC_NR];
87
88extern enum pcpu_fc pcpu_chosen_fc;
89
3cbc8565
TH		 90typedef void * (*pcpu_fc_alloc_fn_t)(unsigned int cpu, size_t size,
91 size_t align);
d4b95f80
TH		 92typedef void (*pcpu_fc_free_fn_t)(void *ptr, size_t size);
93typedef void (*pcpu_fc_populate_pte_fn_t)(unsigned long addr);
a530b795 94typedef int (pcpu_fc_cpu_distance_fn_t)(unsigned int from, unsigned int to);
fbf59bc9 95
fd1e8a1f
TH		 96extern struct pcpu_alloc_info * __init pcpu_alloc_alloc_info(int nr_groups,
97 int nr_units);
98extern void __init pcpu_free_alloc_info(struct pcpu_alloc_info *ai);
99
100extern struct pcpu_alloc_info * __init pcpu_build_alloc_info(
101 size_t reserved_size, ssize_t dyn_size,
102 size_t atom_size,
033e48fb 103 pcpu_fc_cpu_distance_fn_t cpu_distance_fn);
033e48fb 104
fb435d52
TH		 105extern int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
106 void *base_addr);
8d408b4b 107
08fc4580 108#ifdef CONFIG_NEED_PER_CPU_EMBED_FIRST_CHUNK
c8826dd5
TH		 109extern int __init pcpu_embed_first_chunk(size_t reserved_size, ssize_t dyn_size,
110 size_t atom_size,
111 pcpu_fc_cpu_distance_fn_t cpu_distance_fn,
112 pcpu_fc_alloc_fn_t alloc_fn,
113 pcpu_fc_free_fn_t free_fn);
08fc4580 114#endif
66c3a757 115
08fc4580 116#ifdef CONFIG_NEED_PER_CPU_PAGE_FIRST_CHUNK
fb435d52 117extern int __init pcpu_page_first_chunk(size_t reserved_size,
d4b95f80
TH		 118 pcpu_fc_alloc_fn_t alloc_fn,
119 pcpu_fc_free_fn_t free_fn,
120 pcpu_fc_populate_pte_fn_t populate_pte_fn);
08fc4580 121#endif
d4b95f80 122
f2a8205c
TH		 123/*
124 * Use this to get to a cpu's version of the per-cpu object
125 * dynamically allocated. Non-atomic access to the current CPU's
126 * version should probably be combined with get_cpu()/put_cpu().
127 */
fbf59bc9
TH		 128#define per_cpu_ptr(ptr, cpu) SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu)))
129
edcb4639 130extern void *__alloc_reserved_percpu(size_t size, size_t align);
f2a8205c
TH		 131extern void *__alloc_percpu(size_t size, size_t align);
132extern void free_percpu(void *__pdata);
1da177e4 133
e74e3962
TH		 134#ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA
135extern void __init setup_per_cpu_areas(void);
136#endif
137
1da177e4
LT		 138#else /* CONFIG_SMP */
139
b36128c8 140#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); })
7ff6f082 141
f2a8205c 142static inline void *__alloc_percpu(size_t size, size_t align)
7ff6f082 143{
f2a8205c
TH		 144 /*
145 * Can't easily make larger alignment work with kmalloc. WARN
146 * on it. Larger alignment should only be used for module
147 * percpu sections on SMP for which this path isn't used.
148 */
e3176036 149 WARN_ON_ONCE(align > SMP_CACHE_BYTES);
d2b02615 150 return kzalloc(size, GFP_KERNEL);
7ff6f082
MP		 151}
152
f2a8205c 153static inline void free_percpu(void *p)
7ff6f082 154{
f2a8205c 155 kfree(p);
1da177e4
LT		 156}
157
e74e3962
TH		 158static inline void __init setup_per_cpu_areas(void) { }
159
a76761b6
TH		 160static inline void *pcpu_lpage_remapped(void *kaddr)
161{
162 return NULL;
163}
164
1da177e4
LT		 165#endif /* CONFIG_SMP */
166
64ef291f
TH		 167#define alloc_percpu(type) \
168 (typeof(type) *)__alloc_percpu(sizeof(type), __alignof__(type))
1da177e4 169
066123a5
TH		 170/*
171 * Optional methods for optimized non-lvalue per-cpu variable access.
172 *
173 * @var can be a percpu variable or a field of it and its size should
174 * equal char, int or long. percpu_read() evaluates to a lvalue and
175 * all others to void.
176 *
177 * These operations are guaranteed to be atomic w.r.t. preemption.
178 * The generic versions use plain get/put_cpu_var(). Archs are
179 * encouraged to implement single-instruction alternatives which don't
180 * require preemption protection.
181 */
182#ifndef percpu_read
183# define percpu_read(var) \
184 ({ \
dd17c8f7 185 typeof(var) __tmp_var__; \
066123a5
TH		 186 __tmp_var__ = get_cpu_var(var); \
187 put_cpu_var(var); \
188 __tmp_var__; \
189 })
190#endif
191
192#define __percpu_generic_to_op(var, val, op) \
193do { \
194 get_cpu_var(var) op val; \
195 put_cpu_var(var); \
196} while (0)
197
198#ifndef percpu_write
199# define percpu_write(var, val) __percpu_generic_to_op(var, (val), =)
200#endif
201
202#ifndef percpu_add
203# define percpu_add(var, val) __percpu_generic_to_op(var, (val), +=)
204#endif
205
206#ifndef percpu_sub
207# define percpu_sub(var, val) __percpu_generic_to_op(var, (val), -=)
208#endif
209
210#ifndef percpu_and
211# define percpu_and(var, val) __percpu_generic_to_op(var, (val), &=)
212#endif
213
214#ifndef percpu_or
215# define percpu_or(var, val) __percpu_generic_to_op(var, (val), |=)
216#endif
217
218#ifndef percpu_xor
219# define percpu_xor(var, val) __percpu_generic_to_op(var, (val), ^=)
220#endif
221
7340a0b1
CL		 222/*
223 * Branching function to split up a function into a set of functions that
224 * are called for different scalar sizes of the objects handled.
225 */
226
227extern void __bad_size_call_parameter(void);
228
0f5e4816
TH		 229#define __pcpu_size_call_return(stem, variable) \
230({ typeof(variable) pscr_ret__; \
7340a0b1 231 switch(sizeof(variable)) { \
0f5e4816
TH		 232 case 1: pscr_ret__ = stem##1(variable);break; \
233 case 2: pscr_ret__ = stem##2(variable);break; \
234 case 4: pscr_ret__ = stem##4(variable);break; \
235 case 8: pscr_ret__ = stem##8(variable);break; \
7340a0b1
CL		 236 default: \
237 __bad_size_call_parameter();break; \
238 } \
0f5e4816 239 pscr_ret__; \
7340a0b1
CL		 240})
241
0f5e4816 242#define __pcpu_size_call(stem, variable, ...) \
7340a0b1
CL		 243do { \
244 switch(sizeof(variable)) { \
245 case 1: stem##1(variable, __VA_ARGS__);break; \
246 case 2: stem##2(variable, __VA_ARGS__);break; \
247 case 4: stem##4(variable, __VA_ARGS__);break; \
248 case 8: stem##8(variable, __VA_ARGS__);break; \
249 default: \
250 __bad_size_call_parameter();break; \
251 } \
252} while (0)
253
254/*
255 * Optimized manipulation for memory allocated through the per cpu
dd17c8f7 256 * allocator or for addresses of per cpu variables.
7340a0b1
CL		 257 *
258 * These operation guarantee exclusivity of access for other operations
259 * on the *same* processor. The assumption is that per cpu data is only
260 * accessed by a single processor instance (the current one).
261 *
262 * The first group is used for accesses that must be done in a
263 * preemption safe way since we know that the context is not preempt
264 * safe. Interrupts may occur. If the interrupt modifies the variable
265 * too then RMW actions will not be reliable.
266 *
267 * The arch code can provide optimized functions in two ways:
268 *
269 * 1. Override the function completely. F.e. define this_cpu_add().
270 * The arch must then ensure that the various scalar format passed
271 * are handled correctly.
272 *
273 * 2. Provide functions for certain scalar sizes. F.e. provide
274 * this_cpu_add_2() to provide per cpu atomic operations for 2 byte
275 * sized RMW actions. If arch code does not provide operations for
276 * a scalar size then the fallback in the generic code will be
277 * used.
278 */
279
280#define _this_cpu_generic_read(pcp) \
281({ typeof(pcp) ret__; \
282 preempt_disable(); \
283 ret__ = *this_cpu_ptr(&(pcp)); \
284 preempt_enable(); \
285 ret__; \
286})
287
288#ifndef this_cpu_read
289# ifndef this_cpu_read_1
290# define this_cpu_read_1(pcp) _this_cpu_generic_read(pcp)
291# endif
292# ifndef this_cpu_read_2
293# define this_cpu_read_2(pcp) _this_cpu_generic_read(pcp)
294# endif
295# ifndef this_cpu_read_4
296# define this_cpu_read_4(pcp) _this_cpu_generic_read(pcp)
297# endif
298# ifndef this_cpu_read_8
299# define this_cpu_read_8(pcp) _this_cpu_generic_read(pcp)
300# endif
0f5e4816 301# define this_cpu_read(pcp) __pcpu_size_call_return(this_cpu_read_, (pcp))
7340a0b1
CL		 302#endif
303
304#define _this_cpu_generic_to_op(pcp, val, op) \
305do { \
306 preempt_disable(); \
307 *__this_cpu_ptr(&pcp) op val; \
308 preempt_enable(); \
309} while (0)
310
311#ifndef this_cpu_write
312# ifndef this_cpu_write_1
313# define this_cpu_write_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
314# endif
315# ifndef this_cpu_write_2
316# define this_cpu_write_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
317# endif
318# ifndef this_cpu_write_4
319# define this_cpu_write_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
320# endif
321# ifndef this_cpu_write_8
322# define this_cpu_write_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), =)
323# endif
0f5e4816 324# define this_cpu_write(pcp, val) __pcpu_size_call(this_cpu_write_, (pcp), (val))
7340a0b1
CL		 325#endif
326
327#ifndef this_cpu_add
328# ifndef this_cpu_add_1
329# define this_cpu_add_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
330# endif
331# ifndef this_cpu_add_2
332# define this_cpu_add_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
333# endif
334# ifndef this_cpu_add_4
335# define this_cpu_add_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
336# endif
337# ifndef this_cpu_add_8
338# define this_cpu_add_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), +=)
339# endif
0f5e4816 340# define this_cpu_add(pcp, val) __pcpu_size_call(this_cpu_add_, (pcp), (val))
7340a0b1
CL		 341#endif
342
343#ifndef this_cpu_sub
344# define this_cpu_sub(pcp, val) this_cpu_add((pcp), -(val))
345#endif
346
347#ifndef this_cpu_inc
348# define this_cpu_inc(pcp) this_cpu_add((pcp), 1)
349#endif
350
351#ifndef this_cpu_dec
352# define this_cpu_dec(pcp) this_cpu_sub((pcp), 1)
353#endif
354
355#ifndef this_cpu_and
356# ifndef this_cpu_and_1
357# define this_cpu_and_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
358# endif
359# ifndef this_cpu_and_2
360# define this_cpu_and_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
361# endif
362# ifndef this_cpu_and_4
363# define this_cpu_and_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
364# endif
365# ifndef this_cpu_and_8
366# define this_cpu_and_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), &=)
367# endif
0f5e4816 368# define this_cpu_and(pcp, val) __pcpu_size_call(this_cpu_and_, (pcp), (val))
7340a0b1
CL		 369#endif
370
371#ifndef this_cpu_or
372# ifndef this_cpu_or_1
373# define this_cpu_or_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
374# endif
375# ifndef this_cpu_or_2
376# define this_cpu_or_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
377# endif
378# ifndef this_cpu_or_4
379# define this_cpu_or_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
380# endif
381# ifndef this_cpu_or_8
382# define this_cpu_or_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), |=)
383# endif
0f5e4816 384# define this_cpu_or(pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val))
7340a0b1
CL		 385#endif
386
387#ifndef this_cpu_xor
388# ifndef this_cpu_xor_1
389# define this_cpu_xor_1(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=)
390# endif
391# ifndef this_cpu_xor_2
392# define this_cpu_xor_2(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=)
393# endif
394# ifndef this_cpu_xor_4
395# define this_cpu_xor_4(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=)
396# endif
397# ifndef this_cpu_xor_8
398# define this_cpu_xor_8(pcp, val) _this_cpu_generic_to_op((pcp), (val), ^=)
399# endif
0f5e4816 400# define this_cpu_xor(pcp, val) __pcpu_size_call(this_cpu_or_, (pcp), (val))
7340a0b1
CL		 401#endif
402
403/*
404 * Generic percpu operations that do not require preemption handling.
405 * Either we do not care about races or the caller has the
406 * responsibility of handling preemptions issues. Arch code can still
407 * override these instructions since the arch per cpu code may be more
408 * efficient and may actually get race freeness for free (that is the
409 * case for x86 for example).
410 *
411 * If there is no other protection through preempt disable and/or
412 * disabling interupts then one of these RMW operations can show unexpected
413 * behavior because the execution thread was rescheduled on another processor
414 * or an interrupt occurred and the same percpu variable was modified from
415 * the interrupt context.
416 */
417#ifndef __this_cpu_read
418# ifndef __this_cpu_read_1
419# define __this_cpu_read_1(pcp) (*__this_cpu_ptr(&(pcp)))
420# endif
421# ifndef __this_cpu_read_2
422# define __this_cpu_read_2(pcp) (*__this_cpu_ptr(&(pcp)))
423# endif
424# ifndef __this_cpu_read_4
425# define __this_cpu_read_4(pcp) (*__this_cpu_ptr(&(pcp)))
426# endif
427# ifndef __this_cpu_read_8
428# define __this_cpu_read_8(pcp) (*__this_cpu_ptr(&(pcp)))
429# endif
0f5e4816 430# define __this_cpu_read(pcp) __pcpu_size_call_return(__this_cpu_read_, (pcp))
7340a0b1
CL		 431#endif
432
433#define __this_cpu_generic_to_op(pcp, val, op) \
434do { \
435 *__this_cpu_ptr(&(pcp)) op val; \
436} while (0)
437
438#ifndef __this_cpu_write
439# ifndef __this_cpu_write_1
440# define __this_cpu_write_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
441# endif
442# ifndef __this_cpu_write_2
443# define __this_cpu_write_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
444# endif
445# ifndef __this_cpu_write_4
446# define __this_cpu_write_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
447# endif
448# ifndef __this_cpu_write_8
449# define __this_cpu_write_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), =)
450# endif
0f5e4816 451# define __this_cpu_write(pcp, val) __pcpu_size_call(__this_cpu_write_, (pcp), (val))
7340a0b1
CL		 452#endif
453
454#ifndef __this_cpu_add
455# ifndef __this_cpu_add_1
456# define __this_cpu_add_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
457# endif
458# ifndef __this_cpu_add_2
459# define __this_cpu_add_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
460# endif
461# ifndef __this_cpu_add_4
462# define __this_cpu_add_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
463# endif
464# ifndef __this_cpu_add_8
465# define __this_cpu_add_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), +=)
466# endif
0f5e4816 467# define __this_cpu_add(pcp, val) __pcpu_size_call(__this_cpu_add_, (pcp), (val))
7340a0b1
CL		 468#endif
469
470#ifndef __this_cpu_sub
471# define __this_cpu_sub(pcp, val) __this_cpu_add((pcp), -(val))
472#endif
473
474#ifndef __this_cpu_inc
475# define __this_cpu_inc(pcp) __this_cpu_add((pcp), 1)
476#endif
477
478#ifndef __this_cpu_dec
479# define __this_cpu_dec(pcp) __this_cpu_sub((pcp), 1)
480#endif
481
482#ifndef __this_cpu_and
483# ifndef __this_cpu_and_1
484# define __this_cpu_and_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
485# endif
486# ifndef __this_cpu_and_2
487# define __this_cpu_and_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
488# endif
489# ifndef __this_cpu_and_4
490# define __this_cpu_and_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
491# endif
492# ifndef __this_cpu_and_8
493# define __this_cpu_and_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), &=)
494# endif
0f5e4816 495# define __this_cpu_and(pcp, val) __pcpu_size_call(__this_cpu_and_, (pcp), (val))
7340a0b1
CL		 496#endif
497
498#ifndef __this_cpu_or
499# ifndef __this_cpu_or_1
500# define __this_cpu_or_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=)
501# endif
502# ifndef __this_cpu_or_2
503# define __this_cpu_or_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=)
504# endif
505# ifndef __this_cpu_or_4
506# define __this_cpu_or_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=)
507# endif
508# ifndef __this_cpu_or_8
509# define __this_cpu_or_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), |=)
510# endif
0f5e4816 511# define __this_cpu_or(pcp, val) __pcpu_size_call(__this_cpu_or_, (pcp), (val))
7340a0b1
CL		 512#endif
513
514#ifndef __this_cpu_xor
515# ifndef __this_cpu_xor_1
516# define __this_cpu_xor_1(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=)
517# endif
518# ifndef __this_cpu_xor_2
519# define __this_cpu_xor_2(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=)
520# endif
521# ifndef __this_cpu_xor_4
522# define __this_cpu_xor_4(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=)
523# endif
524# ifndef __this_cpu_xor_8
525# define __this_cpu_xor_8(pcp, val) __this_cpu_generic_to_op((pcp), (val), ^=)
526# endif
0f5e4816 527# define __this_cpu_xor(pcp, val) __pcpu_size_call(__this_cpu_xor_, (pcp), (val))
7340a0b1
CL		 528#endif
529
530/*
531 * IRQ safe versions of the per cpu RMW operations. Note that these operations
532 * are *not* safe against modification of the same variable from another
533 * processors (which one gets when using regular atomic operations)
534 . They are guaranteed to be atomic vs. local interrupts and
535 * preemption only.
536 */
537#define irqsafe_cpu_generic_to_op(pcp, val, op) \
538do { \
539 unsigned long flags; \
540 local_irq_save(flags); \
541 *__this_cpu_ptr(&(pcp)) op val; \
542 local_irq_restore(flags); \
543} while (0)
544
545#ifndef irqsafe_cpu_add
546# ifndef irqsafe_cpu_add_1
547# define irqsafe_cpu_add_1(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), +=)
548# endif
549# ifndef irqsafe_cpu_add_2
550# define irqsafe_cpu_add_2(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), +=)
551# endif
552# ifndef irqsafe_cpu_add_4
553# define irqsafe_cpu_add_4(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), +=)
554# endif
555# ifndef irqsafe_cpu_add_8
556# define irqsafe_cpu_add_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), +=)
557# endif
0f5e4816 558# define irqsafe_cpu_add(pcp, val) __pcpu_size_call(irqsafe_cpu_add_, (pcp), (val))
7340a0b1
CL		 559#endif
560
561#ifndef irqsafe_cpu_sub
562# define irqsafe_cpu_sub(pcp, val) irqsafe_cpu_add((pcp), -(val))
563#endif
564
565#ifndef irqsafe_cpu_inc
566# define irqsafe_cpu_inc(pcp) irqsafe_cpu_add((pcp), 1)
567#endif
568
569#ifndef irqsafe_cpu_dec
570# define irqsafe_cpu_dec(pcp) irqsafe_cpu_sub((pcp), 1)
571#endif
572
573#ifndef irqsafe_cpu_and
574# ifndef irqsafe_cpu_and_1
575# define irqsafe_cpu_and_1(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), &=)
576# endif
577# ifndef irqsafe_cpu_and_2
578# define irqsafe_cpu_and_2(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), &=)
579# endif
580# ifndef irqsafe_cpu_and_4
581# define irqsafe_cpu_and_4(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), &=)
582# endif
583# ifndef irqsafe_cpu_and_8
584# define irqsafe_cpu_and_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), &=)
585# endif
0f5e4816 586# define irqsafe_cpu_and(pcp, val) __pcpu_size_call(irqsafe_cpu_and_, (val))
7340a0b1
CL		 587#endif
588
589#ifndef irqsafe_cpu_or
590# ifndef irqsafe_cpu_or_1
591# define irqsafe_cpu_or_1(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), |=)
592# endif
593# ifndef irqsafe_cpu_or_2
594# define irqsafe_cpu_or_2(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), |=)
595# endif
596# ifndef irqsafe_cpu_or_4
597# define irqsafe_cpu_or_4(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), |=)
598# endif
599# ifndef irqsafe_cpu_or_8
600# define irqsafe_cpu_or_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), |=)
601# endif
0f5e4816 602# define irqsafe_cpu_or(pcp, val) __pcpu_size_call(irqsafe_cpu_or_, (val))
7340a0b1
CL		 603#endif
604
605#ifndef irqsafe_cpu_xor
606# ifndef irqsafe_cpu_xor_1
607# define irqsafe_cpu_xor_1(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), ^=)
608# endif
609# ifndef irqsafe_cpu_xor_2
610# define irqsafe_cpu_xor_2(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), ^=)
611# endif
612# ifndef irqsafe_cpu_xor_4
613# define irqsafe_cpu_xor_4(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), ^=)
614# endif
615# ifndef irqsafe_cpu_xor_8
616# define irqsafe_cpu_xor_8(pcp, val) irqsafe_cpu_generic_to_op((pcp), (val), ^=)
617# endif
0f5e4816 618# define irqsafe_cpu_xor(pcp, val) __pcpu_size_call(irqsafe_cpu_xor_, (val))
7340a0b1
CL		 619#endif
620
1da177e4 621#endif /* __LINUX_PERCPU_H */
Linux kernel - Deliverables
This page took 0.646719 seconds and 5 git commands to generate.