Commit | Line | Data |
---|---|---|
ef6695f1 MD |
1 | // SPDX-License-Identifier: MIT |
2 | // SPDX-FileCopyrightText: 2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> | |
3 | ||
34337fec | 4 | #include <rseq/mempool.h> |
ef6695f1 MD |
5 | #include <sys/mman.h> |
6 | #include <assert.h> | |
7 | #include <string.h> | |
8 | #include <pthread.h> | |
9 | #include <unistd.h> | |
10 | #include <stdlib.h> | |
11 | #include <rseq/compiler.h> | |
12 | #include <errno.h> | |
13 | #include <stdint.h> | |
14 | #include <stdbool.h> | |
367e559c MD |
15 | #include <stdio.h> |
16 | ||
17 | #ifdef HAVE_LIBNUMA | |
18 | # include <numa.h> | |
19 | # include <numaif.h> | |
20 | #endif | |
ef6695f1 | 21 | |
34337fec | 22 | #include "rseq-utils.h" |
cb475906 | 23 | #include "smp.h" |
19be9217 | 24 | |
ef6695f1 | 25 | /* |
b73b0c25 | 26 | * rseq-mempool.c: rseq CPU-Local Storage (CLS) memory allocator. |
ef6695f1 | 27 | * |
8ab16a24 MD |
28 | * The rseq per-CPU memory allocator allows the application the request |
29 | * memory pools of CPU-Local memory each of containing objects of a | |
8aa1462d MD |
30 | * given size (rounded to next power of 2), reserving a given virtual |
31 | * address size per CPU, for a given maximum number of CPUs. | |
8ab16a24 MD |
32 | * |
33 | * The per-CPU memory allocator is analogous to TLS (Thread-Local | |
34 | * Storage) memory: TLS is Thread-Local Storage, whereas the per-CPU | |
35 | * memory allocator provides CPU-Local Storage. | |
ef6695f1 MD |
36 | */ |
37 | ||
3236da62 | 38 | #define POOL_SET_NR_ENTRIES RSEQ_BITS_PER_LONG |
ef6695f1 | 39 | |
72b100a1 MD |
40 | /* |
41 | * Smallest allocation should hold enough space for a free list pointer. | |
42 | */ | |
ef6695f1 MD |
43 | #if RSEQ_BITS_PER_LONG == 64 |
44 | # define POOL_SET_MIN_ENTRY 3 /* Smallest item_len=8 */ | |
45 | #else | |
46 | # define POOL_SET_MIN_ENTRY 2 /* Smallest item_len=4 */ | |
47 | #endif | |
48 | ||
bb1552e2 MD |
49 | /* |
50 | * Skip pool index 0 to ensure allocated entries at index 0 do not match | |
51 | * a NULL pointer. | |
52 | */ | |
53 | #define FIRST_POOL 1 | |
54 | ||
0fdf7a4c OD |
55 | #define BIT_PER_ULONG (8 * sizeof(unsigned long)) |
56 | ||
57d8b586 OD |
57 | #define MOVE_PAGES_BATCH_SIZE 4096 |
58 | ||
0ba2a93e | 59 | #define RANGE_HEADER_OFFSET sizeof(struct rseq_mempool_range) |
4aa3220c | 60 | |
ef6695f1 MD |
61 | struct free_list_node; |
62 | ||
63 | struct free_list_node { | |
64 | struct free_list_node *next; | |
65 | }; | |
66 | ||
cb475906 MD |
67 | enum mempool_type { |
68 | MEMPOOL_TYPE_PERCPU = 0, /* Default */ | |
69 | MEMPOOL_TYPE_GLOBAL = 1, | |
70 | }; | |
71 | ||
0ba2a93e | 72 | struct rseq_mempool_attr { |
a82006d0 | 73 | bool mmap_set; |
9bd07c29 MD |
74 | void *(*mmap_func)(void *priv, size_t len); |
75 | int (*munmap_func)(void *priv, void *ptr, size_t len); | |
76 | void *mmap_priv; | |
d6acc8aa MD |
77 | |
78 | bool robust_set; | |
cb475906 MD |
79 | |
80 | enum mempool_type type; | |
81 | size_t stride; | |
82 | int max_nr_cpus; | |
9bd07c29 MD |
83 | }; |
84 | ||
0ba2a93e | 85 | struct rseq_mempool_range; |
b73b0c25 | 86 | |
0ba2a93e MD |
87 | struct rseq_mempool_range { |
88 | struct rseq_mempool_range *next; | |
89 | struct rseq_mempool *pool; /* Backward ref. to container pool. */ | |
4aa3220c | 90 | void *header; |
ef6695f1 | 91 | void *base; |
b73b0c25 MD |
92 | size_t next_unused; |
93 | /* Track alloc/free. */ | |
94 | unsigned long *alloc_bitmap; | |
95 | }; | |
96 | ||
0ba2a93e | 97 | struct rseq_mempool { |
b73b0c25 | 98 | /* Linked-list of ranges. */ |
0ba2a93e | 99 | struct rseq_mempool_range *ranges; |
b73b0c25 | 100 | |
ef6695f1 | 101 | size_t item_len; |
ef6695f1 | 102 | int item_order; |
ef6695f1 MD |
103 | |
104 | /* | |
8ab16a24 | 105 | * The free list chains freed items on the CPU 0 address range. |
ef6695f1 | 106 | * We should rethink this decision if false sharing between |
8ab16a24 | 107 | * malloc/free from other CPUs and data accesses from CPU 0 |
ef6695f1 MD |
108 | * becomes an issue. This is a NULL-terminated singly-linked |
109 | * list. | |
110 | */ | |
111 | struct free_list_node *free_list_head; | |
b73b0c25 | 112 | |
ef6695f1 MD |
113 | /* This lock protects allocation/free within the pool. */ |
114 | pthread_mutex_t lock; | |
9bd07c29 | 115 | |
0ba2a93e | 116 | struct rseq_mempool_attr attr; |
ca452fee | 117 | char *name; |
ef6695f1 MD |
118 | }; |
119 | ||
ef6695f1 MD |
120 | /* |
121 | * Pool set entries are indexed by item_len rounded to the next power of | |
122 | * 2. A pool set can contain NULL pool entries, in which case the next | |
123 | * large enough entry will be used for allocation. | |
124 | */ | |
0ba2a93e | 125 | struct rseq_mempool_set { |
ef6695f1 MD |
126 | /* This lock protects add vs malloc/zmalloc within the pool set. */ |
127 | pthread_mutex_t lock; | |
0ba2a93e | 128 | struct rseq_mempool *entries[POOL_SET_NR_ENTRIES]; |
ef6695f1 MD |
129 | }; |
130 | ||
367e559c | 131 | static |
0ba2a93e | 132 | void *__rseq_pool_percpu_ptr(struct rseq_mempool *pool, int cpu, |
f2981623 | 133 | uintptr_t item_offset, size_t stride) |
367e559c | 134 | { |
b73b0c25 | 135 | /* TODO: Implement multi-ranges support. */ |
f2981623 | 136 | return pool->ranges->base + (stride * cpu) + item_offset; |
367e559c MD |
137 | } |
138 | ||
367e559c | 139 | static |
0ba2a93e | 140 | void rseq_percpu_zero_item(struct rseq_mempool *pool, uintptr_t item_offset) |
367e559c MD |
141 | { |
142 | int i; | |
143 | ||
cb475906 | 144 | for (i = 0; i < pool->attr.max_nr_cpus; i++) { |
f2981623 | 145 | char *p = __rseq_pool_percpu_ptr(pool, i, |
cb475906 | 146 | item_offset, pool->attr.stride); |
367e559c MD |
147 | memset(p, 0, pool->item_len); |
148 | } | |
149 | } | |
150 | ||
b73b0c25 MD |
151 | //TODO: this will need to be reimplemented for ranges, |
152 | //which cannot use __rseq_pool_percpu_ptr. | |
153 | #if 0 //#ifdef HAVE_LIBNUMA | |
154 | static | |
0ba2a93e | 155 | int rseq_mempool_range_init_numa(struct rseq_mempool *pool, struct rseq_mempool_range *range, int numa_flags) |
367e559c | 156 | { |
f2981623 MD |
157 | unsigned long nr_pages, page_len; |
158 | long ret; | |
367e559c MD |
159 | int cpu; |
160 | ||
161 | if (!numa_flags) | |
9bd07c29 | 162 | return 0; |
367e559c | 163 | page_len = rseq_get_page_len(); |
cb475906 MD |
164 | nr_pages = pool->attr.stride >> rseq_get_count_order_ulong(page_len); |
165 | for (cpu = 0; cpu < pool->attr.max_nr_cpus; cpu++) { | |
367e559c | 166 | |
57d8b586 OD |
167 | int status[MOVE_PAGES_BATCH_SIZE]; |
168 | int nodes[MOVE_PAGES_BATCH_SIZE]; | |
169 | void *pages[MOVE_PAGES_BATCH_SIZE]; | |
170 | ||
171 | nodes[0] = numa_node_of_cpu(cpu); | |
172 | for (size_t k = 1; k < RSEQ_ARRAY_SIZE(nodes); ++k) { | |
173 | nodes[k] = nodes[0]; | |
174 | } | |
175 | ||
176 | for (unsigned long page = 0; page < nr_pages;) { | |
177 | ||
178 | size_t max_k = RSEQ_ARRAY_SIZE(pages); | |
179 | size_t left = nr_pages - page; | |
367e559c | 180 | |
57d8b586 OD |
181 | if (left < max_k) { |
182 | max_k = left; | |
183 | } | |
184 | ||
185 | for (size_t k = 0; k < max_k; ++k, ++page) { | |
186 | pages[k] = __rseq_pool_percpu_ptr(pool, cpu, page * page_len); | |
187 | status[k] = -EPERM; | |
188 | } | |
189 | ||
190 | ret = move_pages(0, max_k, pages, nodes, status, numa_flags); | |
191 | ||
192 | if (ret < 0) | |
9bd07c29 | 193 | return ret; |
57d8b586 OD |
194 | |
195 | if (ret > 0) { | |
196 | fprintf(stderr, "%lu pages were not migrated\n", ret); | |
197 | for (size_t k = 0; k < max_k; ++k) { | |
198 | if (status[k] < 0) | |
199 | fprintf(stderr, | |
200 | "Error while moving page %p to numa node %d: %u\n", | |
201 | pages[k], nodes[k], -status[k]); | |
202 | } | |
203 | } | |
367e559c MD |
204 | } |
205 | } | |
9bd07c29 | 206 | return 0; |
367e559c | 207 | } |
b73b0c25 | 208 | |
0ba2a93e | 209 | int rseq_mempool_init_numa(struct rseq_mempool *pool, int numa_flags) |
b73b0c25 | 210 | { |
0ba2a93e | 211 | struct rseq_mempool_range *range; |
b73b0c25 MD |
212 | int ret; |
213 | ||
214 | if (!numa_flags) | |
215 | return 0; | |
216 | for (range = pool->ranges; range; range = range->next) { | |
0ba2a93e | 217 | ret = rseq_mempool_range_init_numa(pool, range, numa_flags); |
b73b0c25 MD |
218 | if (ret) |
219 | return ret; | |
220 | } | |
221 | return 0; | |
222 | } | |
367e559c | 223 | #else |
0ba2a93e | 224 | int rseq_mempool_init_numa(struct rseq_mempool *pool __attribute__((unused)), |
367e559c MD |
225 | int numa_flags __attribute__((unused))) |
226 | { | |
9bd07c29 | 227 | return 0; |
367e559c MD |
228 | } |
229 | #endif | |
230 | ||
9bd07c29 MD |
231 | static |
232 | void *default_mmap_func(void *priv __attribute__((unused)), size_t len) | |
233 | { | |
234 | void *base; | |
235 | ||
236 | base = mmap(NULL, len, PROT_READ | PROT_WRITE, | |
237 | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); | |
238 | if (base == MAP_FAILED) | |
239 | return NULL; | |
240 | return base; | |
241 | } | |
242 | ||
243 | static | |
244 | int default_munmap_func(void *priv __attribute__((unused)), void *ptr, size_t len) | |
245 | { | |
246 | return munmap(ptr, len); | |
247 | } | |
248 | ||
0fdf7a4c | 249 | static |
0ba2a93e | 250 | int create_alloc_bitmap(struct rseq_mempool *pool, struct rseq_mempool_range *range) |
0fdf7a4c OD |
251 | { |
252 | size_t count; | |
253 | ||
cb475906 | 254 | count = ((pool->attr.stride >> pool->item_order) + BIT_PER_ULONG - 1) / BIT_PER_ULONG; |
0fdf7a4c OD |
255 | |
256 | /* | |
9649c7ee MD |
257 | * Not being able to create the validation bitmap is an error |
258 | * that needs to be reported. | |
0fdf7a4c | 259 | */ |
b73b0c25 MD |
260 | range->alloc_bitmap = calloc(count, sizeof(unsigned long)); |
261 | if (!range->alloc_bitmap) | |
9649c7ee MD |
262 | return -1; |
263 | return 0; | |
0fdf7a4c OD |
264 | } |
265 | ||
ca452fee | 266 | static |
0ba2a93e | 267 | const char *get_pool_name(const struct rseq_mempool *pool) |
ca452fee MD |
268 | { |
269 | return pool->name ? : "<anonymous>"; | |
270 | } | |
271 | ||
b73b0c25 | 272 | static |
0ba2a93e | 273 | bool addr_in_pool(const struct rseq_mempool *pool, void *addr) |
b73b0c25 | 274 | { |
0ba2a93e | 275 | struct rseq_mempool_range *range; |
b73b0c25 MD |
276 | |
277 | for (range = pool->ranges; range; range = range->next) { | |
278 | if (addr >= range->base && addr < range->base + range->next_unused) | |
279 | return true; | |
280 | } | |
281 | return false; | |
282 | } | |
283 | ||
a9ec6111 OD |
284 | /* Always inline for __builtin_return_address(0). */ |
285 | static inline __attribute__((always_inline)) | |
0ba2a93e | 286 | void check_free_list(const struct rseq_mempool *pool) |
a9ec6111 | 287 | { |
b73b0c25 MD |
288 | size_t total_item = 0, total_never_allocated = 0, total_freed = 0, |
289 | max_list_traversal = 0, traversal_iteration = 0; | |
0ba2a93e | 290 | struct rseq_mempool_range *range; |
b73b0c25 MD |
291 | |
292 | if (!pool->attr.robust_set) | |
293 | return; | |
294 | ||
295 | for (range = pool->ranges; range; range = range->next) { | |
cb475906 MD |
296 | total_item += pool->attr.stride >> pool->item_order; |
297 | total_never_allocated += (pool->attr.stride - range->next_unused) >> pool->item_order; | |
b73b0c25 MD |
298 | } |
299 | max_list_traversal = total_item - total_never_allocated; | |
a9ec6111 OD |
300 | |
301 | for (struct free_list_node *node = pool->free_list_head, *prev = NULL; | |
302 | node; | |
303 | prev = node, | |
304 | node = node->next) { | |
305 | ||
306 | void *node_addr = node; | |
307 | ||
308 | if (traversal_iteration >= max_list_traversal) { | |
ca452fee MD |
309 | fprintf(stderr, "%s: Corrupted free-list; Possibly infinite loop in pool \"%s\" (%p), caller %p.\n", |
310 | __func__, get_pool_name(pool), pool, __builtin_return_address(0)); | |
a9ec6111 OD |
311 | abort(); |
312 | } | |
313 | ||
314 | /* Node is out of range. */ | |
b73b0c25 | 315 | if (!addr_in_pool(pool, node_addr)) { |
a9ec6111 | 316 | if (prev) |
ca452fee MD |
317 | fprintf(stderr, "%s: Corrupted free-list node %p -> [out-of-range %p] in pool \"%s\" (%p), caller %p.\n", |
318 | __func__, prev, node, get_pool_name(pool), pool, __builtin_return_address(0)); | |
a9ec6111 | 319 | else |
ca452fee MD |
320 | fprintf(stderr, "%s: Corrupted free-list node [out-of-range %p] in pool \"%s\" (%p), caller %p.\n", |
321 | __func__, node, get_pool_name(pool), pool, __builtin_return_address(0)); | |
a9ec6111 OD |
322 | abort(); |
323 | } | |
324 | ||
b73b0c25 MD |
325 | traversal_iteration++; |
326 | total_freed++; | |
a9ec6111 OD |
327 | } |
328 | ||
329 | if (total_never_allocated + total_freed != total_item) { | |
ca452fee MD |
330 | fprintf(stderr, "%s: Corrupted free-list in pool \"%s\" (%p); total-item: %zu total-never-used: %zu total-freed: %zu, caller %p.\n", |
331 | __func__, get_pool_name(pool), pool, total_item, total_never_allocated, total_freed, __builtin_return_address(0)); | |
a9ec6111 OD |
332 | abort(); |
333 | } | |
a9ec6111 OD |
334 | } |
335 | ||
e7cbbc10 MD |
336 | /* Always inline for __builtin_return_address(0). */ |
337 | static inline __attribute__((always_inline)) | |
0ba2a93e | 338 | void destroy_alloc_bitmap(struct rseq_mempool *pool, struct rseq_mempool_range *range) |
0fdf7a4c | 339 | { |
b73b0c25 | 340 | unsigned long *bitmap = range->alloc_bitmap; |
9649c7ee | 341 | size_t count, total_leaks = 0; |
0fdf7a4c | 342 | |
9649c7ee | 343 | if (!bitmap) |
0fdf7a4c | 344 | return; |
0fdf7a4c | 345 | |
cb475906 | 346 | count = ((pool->attr.stride >> pool->item_order) + BIT_PER_ULONG - 1) / BIT_PER_ULONG; |
0fdf7a4c OD |
347 | |
348 | /* Assert that all items in the pool were freed. */ | |
9649c7ee MD |
349 | for (size_t k = 0; k < count; ++k) |
350 | total_leaks += rseq_hweight_ulong(bitmap[k]); | |
351 | if (total_leaks) { | |
ca452fee MD |
352 | fprintf(stderr, "%s: Pool \"%s\" (%p) has %zu leaked items on destroy, caller: %p.\n", |
353 | __func__, get_pool_name(pool), pool, total_leaks, (void *) __builtin_return_address(0)); | |
9649c7ee | 354 | abort(); |
0fdf7a4c OD |
355 | } |
356 | ||
357 | free(bitmap); | |
358 | } | |
359 | ||
b73b0c25 MD |
360 | /* Always inline for __builtin_return_address(0). */ |
361 | static inline __attribute__((always_inline)) | |
0ba2a93e MD |
362 | int rseq_mempool_range_destroy(struct rseq_mempool *pool, |
363 | struct rseq_mempool_range *range) | |
b73b0c25 MD |
364 | { |
365 | destroy_alloc_bitmap(pool, range); | |
5c99f3d6 | 366 | /* range is a header located one page before the aligned mapping. */ |
4aa3220c | 367 | return pool->attr.munmap_func(pool->attr.mmap_priv, range->header, |
cb475906 | 368 | (pool->attr.stride * pool->attr.max_nr_cpus) + rseq_get_page_len()); |
5c99f3d6 MD |
369 | } |
370 | ||
371 | /* | |
372 | * Allocate a memory mapping aligned on @alignment, with an optional | |
373 | * @pre_header before the mapping. | |
374 | */ | |
375 | static | |
0ba2a93e | 376 | void *aligned_mmap_anonymous(struct rseq_mempool *pool, |
5c99f3d6 MD |
377 | size_t page_size, size_t len, size_t alignment, |
378 | void **pre_header, size_t pre_header_len) | |
379 | { | |
380 | size_t minimum_page_count, page_count, extra, total_allocate = 0; | |
381 | int page_order; | |
382 | void *ptr; | |
383 | ||
384 | if (len < page_size || alignment < page_size || | |
385 | !is_pow2(len) || !is_pow2(alignment)) { | |
386 | errno = EINVAL; | |
387 | return NULL; | |
388 | } | |
389 | page_order = rseq_get_count_order_ulong(page_size); | |
390 | if (page_order < 0) { | |
391 | errno = EINVAL; | |
392 | return NULL; | |
393 | } | |
394 | if (pre_header_len && (pre_header_len & (page_size - 1))) { | |
395 | errno = EINVAL; | |
396 | return NULL; | |
397 | } | |
398 | ||
399 | minimum_page_count = (pre_header_len + len) >> page_order; | |
400 | page_count = (pre_header_len + len + alignment - page_size) >> page_order; | |
401 | ||
402 | assert(page_count >= minimum_page_count); | |
403 | ||
404 | ptr = pool->attr.mmap_func(pool->attr.mmap_priv, page_count << page_order); | |
405 | if (!ptr) | |
406 | goto alloc_error; | |
407 | ||
408 | total_allocate = page_count << page_order; | |
409 | ||
410 | if (!(((uintptr_t) ptr + pre_header_len) & (alignment - 1))) { | |
411 | /* Pointer is already aligned. ptr points to pre_header. */ | |
412 | goto out; | |
413 | } | |
414 | ||
415 | /* Unmap extra before. */ | |
416 | extra = offset_align((uintptr_t) ptr + pre_header_len, alignment); | |
417 | assert(!(extra & (page_size - 1))); | |
418 | if (pool->attr.munmap_func(pool->attr.mmap_priv, ptr, extra)) { | |
419 | perror("munmap"); | |
420 | abort(); | |
421 | } | |
422 | total_allocate -= extra; | |
423 | ptr += extra; /* ptr points to pre_header */ | |
424 | page_count -= extra >> page_order; | |
425 | out: | |
426 | assert(page_count >= minimum_page_count); | |
427 | ||
428 | if (page_count > minimum_page_count) { | |
429 | void *extra_ptr; | |
430 | ||
431 | /* Unmap extra after. */ | |
432 | extra_ptr = ptr + (minimum_page_count << page_order); | |
433 | extra = (page_count - minimum_page_count) << page_order; | |
434 | if (pool->attr.munmap_func(pool->attr.mmap_priv, extra_ptr, extra)) { | |
435 | perror("munmap"); | |
436 | abort(); | |
437 | } | |
438 | total_allocate -= extra; | |
439 | } | |
440 | ||
441 | assert(!(((uintptr_t)ptr + pre_header_len) & (alignment - 1))); | |
442 | assert(total_allocate == len + pre_header_len); | |
443 | ||
444 | alloc_error: | |
445 | if (ptr) { | |
446 | if (pre_header) | |
447 | *pre_header = ptr; | |
448 | ptr += pre_header_len; | |
449 | } | |
450 | return ptr; | |
b73b0c25 MD |
451 | } |
452 | ||
453 | static | |
0ba2a93e | 454 | struct rseq_mempool_range *rseq_mempool_range_create(struct rseq_mempool *pool) |
b73b0c25 | 455 | { |
0ba2a93e | 456 | struct rseq_mempool_range *range; |
5c99f3d6 | 457 | unsigned long page_size; |
4aa3220c | 458 | void *header; |
b73b0c25 MD |
459 | void *base; |
460 | ||
5c99f3d6 | 461 | page_size = rseq_get_page_len(); |
b73b0c25 | 462 | |
5c99f3d6 | 463 | base = aligned_mmap_anonymous(pool, page_size, |
cb475906 MD |
464 | pool->attr.stride * pool->attr.max_nr_cpus, |
465 | pool->attr.stride, | |
4aa3220c | 466 | &header, page_size); |
b73b0c25 | 467 | if (!base) |
5c99f3d6 | 468 | return NULL; |
0ba2a93e | 469 | range = (struct rseq_mempool_range *) (base - RANGE_HEADER_OFFSET); |
5c99f3d6 | 470 | range->pool = pool; |
b73b0c25 | 471 | range->base = base; |
4aa3220c | 472 | range->header = header; |
b73b0c25 MD |
473 | if (pool->attr.robust_set) { |
474 | if (create_alloc_bitmap(pool, range)) | |
475 | goto error_alloc; | |
476 | } | |
477 | return range; | |
478 | ||
479 | error_alloc: | |
0ba2a93e | 480 | (void) rseq_mempool_range_destroy(pool, range); |
b73b0c25 MD |
481 | return NULL; |
482 | } | |
483 | ||
0ba2a93e | 484 | int rseq_mempool_destroy(struct rseq_mempool *pool) |
9649c7ee | 485 | { |
0ba2a93e | 486 | struct rseq_mempool_range *range, *next_range; |
b73b0c25 | 487 | int ret = 0; |
9649c7ee | 488 | |
f510ddc5 MD |
489 | if (!pool) |
490 | return 0; | |
b73b0c25 MD |
491 | check_free_list(pool); |
492 | /* Iteration safe against removal. */ | |
493 | for (range = pool->ranges; range && (next_range = range->next, 1); range = next_range) { | |
0ba2a93e | 494 | if (rseq_mempool_range_destroy(pool, range)) |
b73b0c25 MD |
495 | goto end; |
496 | /* Update list head to keep list coherent in case of partial failure. */ | |
497 | pool->ranges = next_range; | |
498 | } | |
9649c7ee | 499 | pthread_mutex_destroy(&pool->lock); |
ca452fee | 500 | free(pool->name); |
9649c7ee MD |
501 | memset(pool, 0, sizeof(*pool)); |
502 | end: | |
b73b0c25 | 503 | return ret; |
9649c7ee MD |
504 | } |
505 | ||
0ba2a93e | 506 | struct rseq_mempool *rseq_mempool_create(const char *pool_name, |
cb475906 | 507 | size_t item_len, const struct rseq_mempool_attr *_attr) |
ef6695f1 | 508 | { |
0ba2a93e MD |
509 | struct rseq_mempool *pool; |
510 | struct rseq_mempool_attr attr = {}; | |
ef6695f1 | 511 | int order; |
ef6695f1 MD |
512 | |
513 | /* Make sure each item is large enough to contain free list pointers. */ | |
514 | if (item_len < sizeof(void *)) | |
515 | item_len = sizeof(void *); | |
516 | ||
517 | /* Align item_len on next power of two. */ | |
19be9217 | 518 | order = rseq_get_count_order_ulong(item_len); |
ef6695f1 MD |
519 | if (order < 0) { |
520 | errno = EINVAL; | |
521 | return NULL; | |
522 | } | |
523 | item_len = 1UL << order; | |
524 | ||
a82006d0 MD |
525 | if (_attr) |
526 | memcpy(&attr, _attr, sizeof(attr)); | |
527 | if (!attr.mmap_set) { | |
528 | attr.mmap_func = default_mmap_func; | |
529 | attr.munmap_func = default_munmap_func; | |
530 | attr.mmap_priv = NULL; | |
9bd07c29 | 531 | } |
a82006d0 | 532 | |
cb475906 MD |
533 | switch (attr.type) { |
534 | case MEMPOOL_TYPE_PERCPU: | |
535 | if (attr.max_nr_cpus < 0) { | |
536 | errno = EINVAL; | |
537 | return NULL; | |
538 | } | |
539 | if (attr.max_nr_cpus == 0) { | |
540 | /* Auto-detect */ | |
541 | attr.max_nr_cpus = get_possible_cpus_array_len(); | |
542 | if (attr.max_nr_cpus == 0) { | |
543 | errno = EINVAL; | |
544 | return NULL; | |
545 | } | |
546 | } | |
547 | break; | |
548 | case MEMPOOL_TYPE_GLOBAL: | |
549 | break; | |
550 | } | |
551 | if (!attr.stride) | |
552 | attr.stride = RSEQ_MEMPOOL_STRIDE; /* Use default */ | |
553 | if (item_len > attr.stride || attr.stride < (size_t) rseq_get_page_len() || | |
554 | !is_pow2(attr.stride)) { | |
555 | errno = EINVAL; | |
556 | return NULL; | |
557 | } | |
558 | ||
0ba2a93e | 559 | pool = calloc(1, sizeof(struct rseq_mempool)); |
bc510b60 MD |
560 | if (!pool) |
561 | return NULL; | |
ef6695f1 | 562 | |
b73b0c25 | 563 | memcpy(&pool->attr, &attr, sizeof(attr)); |
ef6695f1 | 564 | pthread_mutex_init(&pool->lock, NULL); |
ef6695f1 MD |
565 | pool->item_len = item_len; |
566 | pool->item_order = order; | |
b73b0c25 MD |
567 | |
568 | //TODO: implement multi-range support. | |
0ba2a93e | 569 | pool->ranges = rseq_mempool_range_create(pool); |
b73b0c25 MD |
570 | if (!pool->ranges) |
571 | goto error_alloc; | |
0fdf7a4c | 572 | |
ca452fee MD |
573 | if (pool_name) { |
574 | pool->name = strdup(pool_name); | |
575 | if (!pool->name) | |
576 | goto error_alloc; | |
577 | } | |
ef6695f1 | 578 | return pool; |
ef6695f1 | 579 | |
9649c7ee | 580 | error_alloc: |
0ba2a93e | 581 | rseq_mempool_destroy(pool); |
9649c7ee MD |
582 | errno = ENOMEM; |
583 | return NULL; | |
ef6695f1 MD |
584 | } |
585 | ||
e7cbbc10 MD |
586 | /* Always inline for __builtin_return_address(0). */ |
587 | static inline __attribute__((always_inline)) | |
0ba2a93e | 588 | void set_alloc_slot(struct rseq_mempool *pool, size_t item_offset) |
0fdf7a4c | 589 | { |
b73b0c25 | 590 | unsigned long *bitmap = pool->ranges->alloc_bitmap; |
9649c7ee | 591 | size_t item_index = item_offset >> pool->item_order; |
0fdf7a4c OD |
592 | unsigned long mask; |
593 | size_t k; | |
594 | ||
9649c7ee | 595 | if (!bitmap) |
0fdf7a4c | 596 | return; |
0fdf7a4c | 597 | |
9649c7ee | 598 | k = item_index / BIT_PER_ULONG; |
0fdf7a4c OD |
599 | mask = 1ULL << (item_index % BIT_PER_ULONG); |
600 | ||
9649c7ee MD |
601 | /* Print error if bit is already set. */ |
602 | if (bitmap[k] & mask) { | |
ca452fee MD |
603 | fprintf(stderr, "%s: Allocator corruption detected for pool: \"%s\" (%p), item offset: %zu, caller: %p.\n", |
604 | __func__, get_pool_name(pool), pool, item_offset, (void *) __builtin_return_address(0)); | |
9649c7ee MD |
605 | abort(); |
606 | } | |
0fdf7a4c OD |
607 | bitmap[k] |= mask; |
608 | } | |
609 | ||
ef6695f1 | 610 | static |
0ba2a93e | 611 | void __rseq_percpu *__rseq_percpu_malloc(struct rseq_mempool *pool, bool zeroed) |
ef6695f1 MD |
612 | { |
613 | struct free_list_node *node; | |
614 | uintptr_t item_offset; | |
d24ee051 | 615 | void __rseq_percpu *addr; |
ef6695f1 MD |
616 | |
617 | pthread_mutex_lock(&pool->lock); | |
618 | /* Get first entry from free list. */ | |
619 | node = pool->free_list_head; | |
620 | if (node != NULL) { | |
621 | /* Remove node from free list (update head). */ | |
622 | pool->free_list_head = node->next; | |
b73b0c25 | 623 | item_offset = (uintptr_t) ((void *) node - pool->ranges->base); |
4aa3220c | 624 | addr = (void __rseq_percpu *) (pool->ranges->base + item_offset); |
ef6695f1 MD |
625 | goto end; |
626 | } | |
cb475906 | 627 | if (pool->ranges->next_unused + pool->item_len > pool->attr.stride) { |
ea1a3ada | 628 | errno = ENOMEM; |
ef6695f1 MD |
629 | addr = NULL; |
630 | goto end; | |
631 | } | |
b73b0c25 | 632 | item_offset = pool->ranges->next_unused; |
4aa3220c | 633 | addr = (void __rseq_percpu *) (pool->ranges->base + item_offset); |
b73b0c25 | 634 | pool->ranges->next_unused += pool->item_len; |
ef6695f1 | 635 | end: |
8f28507f OD |
636 | if (addr) |
637 | set_alloc_slot(pool, item_offset); | |
ef6695f1 MD |
638 | pthread_mutex_unlock(&pool->lock); |
639 | if (zeroed && addr) | |
640 | rseq_percpu_zero_item(pool, item_offset); | |
641 | return addr; | |
642 | } | |
643 | ||
15da5c27 | 644 | void __rseq_percpu *rseq_mempool_percpu_malloc(struct rseq_mempool *pool) |
ef6695f1 MD |
645 | { |
646 | return __rseq_percpu_malloc(pool, false); | |
647 | } | |
648 | ||
15da5c27 | 649 | void __rseq_percpu *rseq_mempool_percpu_zmalloc(struct rseq_mempool *pool) |
ef6695f1 MD |
650 | { |
651 | return __rseq_percpu_malloc(pool, true); | |
652 | } | |
653 | ||
e7cbbc10 MD |
654 | /* Always inline for __builtin_return_address(0). */ |
655 | static inline __attribute__((always_inline)) | |
0ba2a93e | 656 | void clear_alloc_slot(struct rseq_mempool *pool, size_t item_offset) |
0fdf7a4c | 657 | { |
b73b0c25 | 658 | unsigned long *bitmap = pool->ranges->alloc_bitmap; |
9649c7ee | 659 | size_t item_index = item_offset >> pool->item_order; |
0fdf7a4c OD |
660 | unsigned long mask; |
661 | size_t k; | |
662 | ||
9649c7ee | 663 | if (!bitmap) |
0fdf7a4c | 664 | return; |
0fdf7a4c | 665 | |
9649c7ee MD |
666 | k = item_index / BIT_PER_ULONG; |
667 | mask = 1ULL << (item_index % BIT_PER_ULONG); | |
0fdf7a4c | 668 | |
9649c7ee MD |
669 | /* Print error if bit is not set. */ |
670 | if (!(bitmap[k] & mask)) { | |
ca452fee MD |
671 | fprintf(stderr, "%s: Double-free detected for pool: \"%s\" (%p), item offset: %zu, caller: %p.\n", |
672 | __func__, get_pool_name(pool), pool, item_offset, | |
673 | (void *) __builtin_return_address(0)); | |
9649c7ee MD |
674 | abort(); |
675 | } | |
0fdf7a4c OD |
676 | bitmap[k] &= ~mask; |
677 | } | |
678 | ||
cb475906 | 679 | void librseq_mempool_percpu_free(void __rseq_percpu *_ptr, size_t stride) |
ef6695f1 MD |
680 | { |
681 | uintptr_t ptr = (uintptr_t) _ptr; | |
cb475906 | 682 | void *range_base = (void *) (ptr & (~(stride - 1))); |
0ba2a93e MD |
683 | struct rseq_mempool_range *range = (struct rseq_mempool_range *) (range_base - RANGE_HEADER_OFFSET); |
684 | struct rseq_mempool *pool = range->pool; | |
cb475906 | 685 | uintptr_t item_offset = ptr & (stride - 1); |
ef6695f1 MD |
686 | struct free_list_node *head, *item; |
687 | ||
688 | pthread_mutex_lock(&pool->lock); | |
9649c7ee | 689 | clear_alloc_slot(pool, item_offset); |
ef6695f1 MD |
690 | /* Add ptr to head of free list */ |
691 | head = pool->free_list_head; | |
8ab16a24 | 692 | /* Free-list is in CPU 0 range. */ |
4aa3220c | 693 | item = (struct free_list_node *) ptr; |
ef6695f1 MD |
694 | item->next = head; |
695 | pool->free_list_head = item; | |
696 | pthread_mutex_unlock(&pool->lock); | |
697 | } | |
698 | ||
0ba2a93e | 699 | struct rseq_mempool_set *rseq_mempool_set_create(void) |
ef6695f1 | 700 | { |
0ba2a93e | 701 | struct rseq_mempool_set *pool_set; |
ef6695f1 | 702 | |
0ba2a93e | 703 | pool_set = calloc(1, sizeof(struct rseq_mempool_set)); |
ef6695f1 MD |
704 | if (!pool_set) |
705 | return NULL; | |
706 | pthread_mutex_init(&pool_set->lock, NULL); | |
707 | return pool_set; | |
708 | } | |
709 | ||
0ba2a93e | 710 | int rseq_mempool_set_destroy(struct rseq_mempool_set *pool_set) |
ef6695f1 MD |
711 | { |
712 | int order, ret; | |
713 | ||
714 | for (order = POOL_SET_MIN_ENTRY; order < POOL_SET_NR_ENTRIES; order++) { | |
0ba2a93e | 715 | struct rseq_mempool *pool = pool_set->entries[order]; |
ef6695f1 MD |
716 | |
717 | if (!pool) | |
718 | continue; | |
0ba2a93e | 719 | ret = rseq_mempool_destroy(pool); |
ef6695f1 MD |
720 | if (ret) |
721 | return ret; | |
722 | pool_set->entries[order] = NULL; | |
723 | } | |
724 | pthread_mutex_destroy(&pool_set->lock); | |
725 | free(pool_set); | |
726 | return 0; | |
727 | } | |
728 | ||
729 | /* Ownership of pool is handed over to pool set on success. */ | |
0ba2a93e | 730 | int rseq_mempool_set_add_pool(struct rseq_mempool_set *pool_set, struct rseq_mempool *pool) |
ef6695f1 MD |
731 | { |
732 | size_t item_order = pool->item_order; | |
733 | int ret = 0; | |
734 | ||
735 | pthread_mutex_lock(&pool_set->lock); | |
736 | if (pool_set->entries[item_order]) { | |
737 | errno = EBUSY; | |
738 | ret = -1; | |
739 | goto end; | |
740 | } | |
741 | pool_set->entries[pool->item_order] = pool; | |
742 | end: | |
743 | pthread_mutex_unlock(&pool_set->lock); | |
744 | return ret; | |
745 | } | |
746 | ||
747 | static | |
0ba2a93e | 748 | void __rseq_percpu *__rseq_mempool_set_malloc(struct rseq_mempool_set *pool_set, size_t len, bool zeroed) |
ef6695f1 MD |
749 | { |
750 | int order, min_order = POOL_SET_MIN_ENTRY; | |
0ba2a93e | 751 | struct rseq_mempool *pool; |
d24ee051 | 752 | void __rseq_percpu *addr; |
ef6695f1 | 753 | |
d06f5cf5 MD |
754 | order = rseq_get_count_order_ulong(len); |
755 | if (order > POOL_SET_MIN_ENTRY) | |
756 | min_order = order; | |
ef6695f1 MD |
757 | again: |
758 | pthread_mutex_lock(&pool_set->lock); | |
759 | /* First smallest present pool where @len fits. */ | |
760 | for (order = min_order; order < POOL_SET_NR_ENTRIES; order++) { | |
761 | pool = pool_set->entries[order]; | |
762 | ||
763 | if (!pool) | |
764 | continue; | |
765 | if (pool->item_len >= len) | |
766 | goto found; | |
767 | } | |
768 | pool = NULL; | |
769 | found: | |
770 | pthread_mutex_unlock(&pool_set->lock); | |
771 | if (pool) { | |
772 | addr = __rseq_percpu_malloc(pool, zeroed); | |
773 | if (addr == NULL && errno == ENOMEM) { | |
774 | /* | |
775 | * If the allocation failed, try again with a | |
776 | * larger pool. | |
777 | */ | |
778 | min_order = order + 1; | |
779 | goto again; | |
780 | } | |
781 | } else { | |
782 | /* Not found. */ | |
783 | errno = ENOMEM; | |
784 | addr = NULL; | |
785 | } | |
786 | return addr; | |
787 | } | |
788 | ||
15da5c27 | 789 | void __rseq_percpu *rseq_mempool_set_percpu_malloc(struct rseq_mempool_set *pool_set, size_t len) |
ef6695f1 | 790 | { |
0ba2a93e | 791 | return __rseq_mempool_set_malloc(pool_set, len, false); |
ef6695f1 MD |
792 | } |
793 | ||
15da5c27 | 794 | void __rseq_percpu *rseq_mempool_set_percpu_zmalloc(struct rseq_mempool_set *pool_set, size_t len) |
ef6695f1 | 795 | { |
0ba2a93e | 796 | return __rseq_mempool_set_malloc(pool_set, len, true); |
ef6695f1 | 797 | } |
9bd07c29 | 798 | |
0ba2a93e | 799 | struct rseq_mempool_attr *rseq_mempool_attr_create(void) |
a82006d0 | 800 | { |
0ba2a93e | 801 | return calloc(1, sizeof(struct rseq_mempool_attr)); |
a82006d0 MD |
802 | } |
803 | ||
0ba2a93e | 804 | void rseq_mempool_attr_destroy(struct rseq_mempool_attr *attr) |
a82006d0 MD |
805 | { |
806 | free(attr); | |
807 | } | |
808 | ||
0ba2a93e | 809 | int rseq_mempool_attr_set_mmap(struct rseq_mempool_attr *attr, |
a82006d0 | 810 | void *(*mmap_func)(void *priv, size_t len), |
9bd07c29 MD |
811 | int (*munmap_func)(void *priv, void *ptr, size_t len), |
812 | void *mmap_priv) | |
813 | { | |
8118247e MD |
814 | if (!attr) { |
815 | errno = EINVAL; | |
816 | return -1; | |
817 | } | |
a82006d0 | 818 | attr->mmap_set = true; |
9bd07c29 MD |
819 | attr->mmap_func = mmap_func; |
820 | attr->munmap_func = munmap_func; | |
821 | attr->mmap_priv = mmap_priv; | |
8118247e | 822 | return 0; |
9bd07c29 | 823 | } |
d6acc8aa | 824 | |
0ba2a93e | 825 | int rseq_mempool_attr_set_robust(struct rseq_mempool_attr *attr) |
d6acc8aa MD |
826 | { |
827 | if (!attr) { | |
828 | errno = EINVAL; | |
829 | return -1; | |
830 | } | |
831 | attr->robust_set = true; | |
832 | return 0; | |
833 | } | |
cb475906 MD |
834 | |
835 | int rseq_mempool_attr_set_percpu(struct rseq_mempool_attr *attr, | |
836 | size_t stride, int max_nr_cpus) | |
837 | { | |
838 | if (!attr) { | |
839 | errno = EINVAL; | |
840 | return -1; | |
841 | } | |
842 | attr->type = MEMPOOL_TYPE_PERCPU; | |
843 | attr->stride = stride; | |
844 | attr->max_nr_cpus = max_nr_cpus; | |
845 | return 0; | |
846 | } | |
847 | ||
848 | int rseq_mempool_attr_set_global(struct rseq_mempool_attr *attr, | |
849 | size_t stride) | |
850 | { | |
851 | if (!attr) { | |
852 | errno = EINVAL; | |
853 | return -1; | |
854 | } | |
855 | attr->type = MEMPOOL_TYPE_GLOBAL; | |
856 | attr->stride = stride; | |
857 | attr->max_nr_cpus = 1; | |
858 | return 0; | |
859 | } |