Commit | Line | Data |
---|---|---|
2e892f43 CL |
1 | #ifndef _LINUX_SLAB_DEF_H |
2 | #define _LINUX_SLAB_DEF_H | |
3 | ||
4 | /* | |
5 | * Definitions unique to the original Linux SLAB allocator. | |
6 | * | |
7 | * What we provide here is a way to optimize the frequent kmalloc | |
8 | * calls in the kernel by selecting the appropriate general cache | |
9 | * if kmalloc was called with a size that can be established at | |
10 | * compile time. | |
11 | */ | |
12 | ||
13 | #include <linux/init.h> | |
14 | #include <asm/page.h> /* kmalloc_sizes.h needs PAGE_SIZE */ | |
15 | #include <asm/cache.h> /* kmalloc_sizes.h needs L1_CACHE_BYTES */ | |
16 | #include <linux/compiler.h> | |
36994e58 | 17 | #include <trace/kmemtrace.h> |
2e892f43 CL |
18 | |
19 | /* Size description struct for general caches. */ | |
20 | struct cache_sizes { | |
21 | size_t cs_size; | |
22 | struct kmem_cache *cs_cachep; | |
4b51d669 | 23 | #ifdef CONFIG_ZONE_DMA |
2e892f43 | 24 | struct kmem_cache *cs_dmacachep; |
4b51d669 | 25 | #endif |
2e892f43 CL |
26 | }; |
27 | extern struct cache_sizes malloc_sizes[]; | |
28 | ||
6193a2ff PM |
29 | void *kmem_cache_alloc(struct kmem_cache *, gfp_t); |
30 | void *__kmalloc(size_t size, gfp_t flags); | |
31 | ||
36555751 EGM |
32 | #ifdef CONFIG_KMEMTRACE |
33 | extern void *kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags); | |
34 | extern size_t slab_buffer_size(struct kmem_cache *cachep); | |
35 | #else | |
36 | static __always_inline void * | |
37 | kmem_cache_alloc_notrace(struct kmem_cache *cachep, gfp_t flags) | |
2e892f43 | 38 | { |
36555751 EGM |
39 | return kmem_cache_alloc(cachep, flags); |
40 | } | |
41 | static inline size_t slab_buffer_size(struct kmem_cache *cachep) | |
42 | { | |
43 | return 0; | |
44 | } | |
45 | #endif | |
46 | ||
47 | static __always_inline void *kmalloc(size_t size, gfp_t flags) | |
48 | { | |
49 | struct kmem_cache *cachep; | |
50 | void *ret; | |
51 | ||
2e892f43 CL |
52 | if (__builtin_constant_p(size)) { |
53 | int i = 0; | |
6cb8f913 CL |
54 | |
55 | if (!size) | |
56 | return ZERO_SIZE_PTR; | |
57 | ||
2e892f43 CL |
58 | #define CACHE(x) \ |
59 | if (size <= x) \ | |
60 | goto found; \ | |
61 | else \ | |
62 | i++; | |
1c61fc40 | 63 | #include <linux/kmalloc_sizes.h> |
2e892f43 CL |
64 | #undef CACHE |
65 | { | |
66 | extern void __you_cannot_kmalloc_that_much(void); | |
67 | __you_cannot_kmalloc_that_much(); | |
68 | } | |
69 | found: | |
4b51d669 CL |
70 | #ifdef CONFIG_ZONE_DMA |
71 | if (flags & GFP_DMA) | |
36555751 EGM |
72 | cachep = malloc_sizes[i].cs_dmacachep; |
73 | else | |
4b51d669 | 74 | #endif |
36555751 EGM |
75 | cachep = malloc_sizes[i].cs_cachep; |
76 | ||
77 | ret = kmem_cache_alloc_notrace(cachep, flags); | |
78 | ||
79 | kmemtrace_mark_alloc(KMEMTRACE_TYPE_KMALLOC, _THIS_IP_, ret, | |
80 | size, slab_buffer_size(cachep), flags); | |
81 | ||
82 | return ret; | |
2e892f43 CL |
83 | } |
84 | return __kmalloc(size, flags); | |
85 | } | |
86 | ||
2e892f43 CL |
87 | #ifdef CONFIG_NUMA |
88 | extern void *__kmalloc_node(size_t size, gfp_t flags, int node); | |
6193a2ff | 89 | extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node); |
2e892f43 | 90 | |
36555751 EGM |
91 | #ifdef CONFIG_KMEMTRACE |
92 | extern void *kmem_cache_alloc_node_notrace(struct kmem_cache *cachep, | |
93 | gfp_t flags, | |
94 | int nodeid); | |
95 | #else | |
96 | static __always_inline void * | |
97 | kmem_cache_alloc_node_notrace(struct kmem_cache *cachep, | |
98 | gfp_t flags, | |
99 | int nodeid) | |
100 | { | |
101 | return kmem_cache_alloc_node(cachep, flags, nodeid); | |
102 | } | |
103 | #endif | |
104 | ||
105 | static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node) | |
2e892f43 | 106 | { |
36555751 EGM |
107 | struct kmem_cache *cachep; |
108 | void *ret; | |
109 | ||
2e892f43 CL |
110 | if (__builtin_constant_p(size)) { |
111 | int i = 0; | |
6cb8f913 CL |
112 | |
113 | if (!size) | |
114 | return ZERO_SIZE_PTR; | |
115 | ||
2e892f43 CL |
116 | #define CACHE(x) \ |
117 | if (size <= x) \ | |
118 | goto found; \ | |
119 | else \ | |
120 | i++; | |
1c61fc40 | 121 | #include <linux/kmalloc_sizes.h> |
2e892f43 CL |
122 | #undef CACHE |
123 | { | |
124 | extern void __you_cannot_kmalloc_that_much(void); | |
125 | __you_cannot_kmalloc_that_much(); | |
126 | } | |
127 | found: | |
4b51d669 CL |
128 | #ifdef CONFIG_ZONE_DMA |
129 | if (flags & GFP_DMA) | |
36555751 EGM |
130 | cachep = malloc_sizes[i].cs_dmacachep; |
131 | else | |
4b51d669 | 132 | #endif |
36555751 EGM |
133 | cachep = malloc_sizes[i].cs_cachep; |
134 | ||
135 | ret = kmem_cache_alloc_node_notrace(cachep, flags, node); | |
136 | ||
137 | kmemtrace_mark_alloc_node(KMEMTRACE_TYPE_KMALLOC, _THIS_IP_, | |
138 | ret, size, slab_buffer_size(cachep), | |
139 | flags, node); | |
140 | ||
141 | return ret; | |
2e892f43 CL |
142 | } |
143 | return __kmalloc_node(size, flags, node); | |
144 | } | |
145 | ||
146 | #endif /* CONFIG_NUMA */ | |
147 | ||
148 | #endif /* _LINUX_SLAB_DEF_H */ |