include/linux/percpu.h

   1 #ifndef __LINUX_PERCPU_H
   2 #define __LINUX_PERCPU_H
   3
   4 #include <linux/preempt.h>
   5 #include <linux/slab.h> /* For kmalloc() */
   6 #include <linux/smp.h>
   7 #include <linux/cpumask.h>
   8
   9 #include <asm/percpu.h>
  10
  11 #ifndef PER_CPU_BASE_SECTION
  12 #ifdef CONFIG_SMP
  13 #define PER_CPU_BASE_SECTION ".data.percpu"
  14 #else
  15 #define PER_CPU_BASE_SECTION ".data"
  16 #endif
  17 #endif
  18
  19 #ifdef CONFIG_SMP
  20
  21 #ifdef MODULE
  22 #define PER_CPU_SHARED_ALIGNED_SECTION ""
  23 #else
  24 #define PER_CPU_SHARED_ALIGNED_SECTION ".shared_aligned"
  25 #endif
  26 #define PER_CPU_FIRST_SECTION ".first"
  27
  28 #else
  29
  30 #define PER_CPU_SHARED_ALIGNED_SECTION ""
  31 #define PER_CPU_FIRST_SECTION ""
  32
  33 #endif
  34
  35 #define DEFINE_PER_CPU_SECTION(type, name, section)                     \
  36         __attribute__((__section__(PER_CPU_BASE_SECTION section)))      \
  37         PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
  38
  39 #define DEFINE_PER_CPU(type, name)                                      \
  40         DEFINE_PER_CPU_SECTION(type, name, "")
  41
  42 #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name)                       \
  43         DEFINE_PER_CPU_SECTION(type, name, PER_CPU_SHARED_ALIGNED_SECTION) \
  44         ____cacheline_aligned_in_smp
  45
  46 #define DEFINE_PER_CPU_PAGE_ALIGNED(type, name)                         \
  47         DEFINE_PER_CPU_SECTION(type, name, ".page_aligned")
  48
  49 #define DEFINE_PER_CPU_FIRST(type, name)                                \
  50         DEFINE_PER_CPU_SECTION(type, name, PER_CPU_FIRST_SECTION)
  51
  52 #define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
  53 #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
  54
  55 /* Enough to cover all DEFINE_PER_CPUs in kernel, including modules. */
  56 #ifndef PERCPU_ENOUGH_ROOM
  57 #ifdef CONFIG_MODULES
  58 #define PERCPU_MODULE_RESERVE   8192
  59 #else
  60 #define PERCPU_MODULE_RESERVE   0
  61 #endif
  62
  63 #define PERCPU_ENOUGH_ROOM                                              \
  64         (__per_cpu_end - __per_cpu_start + PERCPU_MODULE_RESERVE)
  65 #endif  /* PERCPU_ENOUGH_ROOM */
  66
  67 /*
  68  * Must be an lvalue. Since @var must be a simple identifier,
  69  * we force a syntax error here if it isn't.
  70  */
  71 #define get_cpu_var(var) (*({                           \
  72         extern int simple_identifier_##var(void);       \
  73         preempt_disable();                              \
  74         &__get_cpu_var(var); }))
  75 #define put_cpu_var(var) preempt_enable()
  76
  77 #ifdef CONFIG_SMP
  78
  79 struct percpu_data {
  80         void *ptrs[1];
  81 };
  82
  83 #define __percpu_disguise(pdata) (struct percpu_data *)~(unsigned long)(pdata)
  84
  85 /*
  86  * Use this to get to a cpu's version of the per-cpu object
  87  * dynamically allocated. Non-atomic access to the current CPU's
  88  * version should probably be combined with get_cpu()/put_cpu().
  89  */
  90 #define per_cpu_ptr(ptr, cpu)                                           \
  91 ({                                                                      \
  92         struct percpu_data *__p = __percpu_disguise(ptr);               \
  93         (__typeof__(ptr))__p->ptrs[(cpu)];                              \
  94 })
  95
  96 extern void *__alloc_percpu(size_t size, size_t align);
  97 extern void free_percpu(void *__pdata);
  98
  99 #else /* CONFIG_SMP */
 100
 101 #define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); })
 102
 103 static inline void *__alloc_percpu(size_t size, size_t align)
 104 {
 105         /*
 106          * Can't easily make larger alignment work with kmalloc.  WARN
 107          * on it.  Larger alignment should only be used for module
 108          * percpu sections on SMP for which this path isn't used.
 109          */
 110         WARN_ON_ONCE(align > __alignof__(unsigned long long));
 111         return kzalloc(size, gfp);
 112 }
 113
 114 static inline void free_percpu(void *p)
 115 {
 116         kfree(p);
 117 }
 118
 119 #endif /* CONFIG_SMP */
 120
 121 #define alloc_percpu(type)      (type *)__alloc_percpu(sizeof(type), \
 122                                                        __alignof__(type))
 123
 124 #endif /* __LINUX_PERCPU_H */