1 /* SPDX-License-Identifier: MIT */
2 /* SPDX-FileCopyrightText: 2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
4 #ifndef _RSEQ_PERCPU_ALLOC_H
5 #define _RSEQ_PERCPU_ALLOC_H
12 * rseq/percpu-alloc.h: rseq CPU-Local Storage (CLS) memory allocator.
14 * The rseq per-CPU memory allocator allows the application the request
15 * memory pools of CPU-Local memory each of containing objects of a
16 * given size (rounded to next power of 2), reserving a given virtual
17 * address size per CPU, for a given maximum number of CPUs.
19 * The per-CPU memory allocator is analogous to TLS (Thread-Local
20 * Storage) memory: TLS is Thread-Local Storage, whereas the per-CPU
21 * memory allocator provides CPU-Local Storage.
30 * Tag pointers returned by:
31 * - rseq_percpu_malloc(),
32 * - rseq_percpu_zmalloc(),
33 * - rseq_percpu_pool_set_malloc(),
34 * - rseq_percpu_pool_set_zmalloc().
36 * and passed as parameter to:
37 * - rseq_percpu_ptr(),
38 * - rseq_percpu_free().
40 * with __rseq_percpu for use by static analyzers.
44 struct rseq_percpu_pool
;
47 * rseq_percpu_pool_create: Create a per-cpu memory pool.
49 * Create a per-cpu memory pool for items of size @item_len (rounded to
50 * next power of two). The reserved allocation size is @percpu_len, and
51 * the maximum CPU value expected is (@max_nr_cpus - 1).
53 * Arguments @mmap_prot, @mmap_flags, @mmap_fd, @mmap_offset are passed
54 * as arguments to mmap(2) when allocating the memory area holding the
57 * Argument @numa_flags are passed to move_pages(2). The expected flags
59 * 0: do not move pages to specific numa nodes
60 * (use for e.g. mm_cid indexing).
61 * MPOL_MF_MOVE: move process-private pages to cpu-specific numa nodes.
62 * MPOL_MF_MOVE_ALL: move shared pages to cpu-specific numa nodes
63 * (requires CAP_SYS_NICE).
65 * Returns a pointer to the created percpu pool. Return NULL on error,
66 * with errno set accordingly:
67 * EINVAL: Invalid argument.
68 * ENOMEM: Not enough resources (memory or pool indexes) available to
71 * In addition, if mmap(2) fails, NULL is returned and errno is
72 * propagated from mmap(2).
74 * This API is MT-safe.
76 struct rseq_percpu_pool
*rseq_percpu_pool_create(size_t item_len
,
77 size_t percpu_len
, int max_nr_cpus
,
78 int mmap_prot
, int mmap_flags
, int mmap_fd
, off_t mmap_offset
,
82 * rseq_percpu_pool_destroy: Destroy a per-cpu memory pool.
84 * Destroy a per-cpu memory pool, unmapping its memory and removing the
85 * pool entry from the global index. No pointers allocated from the
86 * pool should be used when it is destroyed. This includes rseq_percpu_ptr().
88 * Argument @pool is a pointer to the per-cpu pool to destroy.
90 * Return values: 0 on success, -1 on error, with errno set accordingly:
91 * ENOENT: Trying to free a pool which was not allocated.
93 * If munmap(2) fails, -1 is returned and errno is propagated from
96 * This API is MT-safe.
98 int rseq_percpu_pool_destroy(struct rseq_percpu_pool
*pool
);
101 * rseq_percpu_malloc: Allocate memory from a per-cpu pool.
103 * Allocate an item from a per-cpu @pool. The allocation will reserve
104 * an item of the size specified by @item_len (rounded to next power of
105 * two) at pool creation. This effectively reserves space for this item
108 * On success, return a "__rseq_percpu" encoded pointer to the pool
109 * item. This encoded pointer is meant to be passed to rseq_percpu_ptr()
110 * to be decoded to a valid address before being accessed.
112 * Return NULL (errno=ENOMEM) if there is not enough space left in the
113 * pool to allocate an item.
115 * This API is MT-safe.
117 void __rseq_percpu
*rseq_percpu_malloc(struct rseq_percpu_pool
*pool
);
120 * rseq_percpu_zmalloc: Allocated zero-initialized memory from a per-cpu pool.
122 * Allocate memory for an item within the pool, and zero-initialize its
123 * memory on all CPUs. See rseq_percpu_malloc for details.
125 * This API is MT-safe.
127 void __rseq_percpu
*rseq_percpu_zmalloc(struct rseq_percpu_pool
*pool
);
130 * rseq_percpu_free: Free memory from a per-cpu pool.
132 * Free an item pointed to by @ptr from its per-cpu pool.
134 * The @ptr argument is a __rseq_percpu encoded pointer returned by
137 * - rseq_percpu_malloc(),
138 * - rseq_percpu_zmalloc(),
139 * - rseq_percpu_pool_set_malloc(),
140 * - rseq_percpu_pool_set_zmalloc().
142 * This API is MT-safe.
144 void rseq_percpu_free(void __rseq_percpu
*ptr
);
147 * rseq_percpu_ptr: Decode a per-cpu pointer.
149 * Decode a per-cpu pointer @ptr to get the associated pointer for the
150 * given @cpu. The @ptr argument is a __rseq_percpu encoded pointer
151 * returned by either:
153 * - rseq_percpu_malloc(),
154 * - rseq_percpu_zmalloc(),
155 * - rseq_percpu_pool_set_malloc(),
156 * - rseq_percpu_pool_set_zmalloc().
158 * The __rseq_percpu pointer can be decoded with rseq_percpu_ptr() even
159 * after it has been freed, as long as its associated pool has not been
160 * destroyed. However, memory pointed to by the decoded pointer should
161 * not be accessed after the __rseq_percpu pointer has been freed.
163 * The macro rseq_percpu_ptr() preserves the type of the @ptr parameter
164 * for the returned pointer, but removes the __rseq_percpu annotation.
166 * This API is MT-safe.
168 void *__rseq_percpu_ptr(void __rseq_percpu
*ptr
, int cpu
);
169 #define rseq_percpu_ptr(ptr, cpu) ((__typeof__(*(ptr)) *) __rseq_percpu_ptr(ptr, cpu))
172 * rseq_percpu_pool_set_create: Create a pool set.
174 * Create a set of pools. Its purpose is to offer a memory allocator API
175 * for variable-length items (e.g. variable length strings). When
176 * created, the pool set has no pool. Pools can be created and added to
177 * the set. One common approach would be to create pools for each
178 * relevant power of two allocation size useful for the application.
179 * Only one pool can be added to the pool set for each power of two
182 * Returns a pool set pointer on success, else returns NULL with
183 * errno=ENOMEM (out of memory).
185 * This API is MT-safe.
187 struct rseq_percpu_pool_set
*rseq_percpu_pool_set_create(void);
190 * rseq_percpu_pool_set_destroy: Destroy a pool set.
192 * Destroy a pool set and its associated resources. The pools that were
193 * added to the pool set are destroyed as well.
195 * Returns 0 on success, -1 on failure (or partial failure), with errno
196 * set by rseq_percpu_pool_destroy(). Using a pool set after destroy
197 * failure is undefined.
199 * This API is MT-safe.
201 int rseq_percpu_pool_set_destroy(struct rseq_percpu_pool_set
*pool_set
);
204 * rseq_percpu_pool_set_add_pool: Add a pool to a pool set.
206 * Add a @pool to the @pool_set. On success, its ownership is handed
207 * over to the pool set, so the caller should not destroy it explicitly.
208 * Only one pool can be added to the pool set for each power of two
211 * Returns 0 on success, -1 on error with the following errno:
212 * - EBUSY: A pool already exists in the pool set for this power of two
215 * This API is MT-safe.
217 int rseq_percpu_pool_set_add_pool(struct rseq_percpu_pool_set
*pool_set
,
218 struct rseq_percpu_pool
*pool
);
221 * rseq_percpu_pool_set_malloc: Allocate memory from a per-cpu pool set.
223 * Allocate an item from a per-cpu @pool. The allocation will reserve
224 * an item of the size specified by @len (rounded to next power of
225 * two). This effectively reserves space for this item on all CPUs.
227 * The space reservation will search for the smallest pool within
228 * @pool_set which respects the following conditions:
230 * - it has an item size large enough to fit @len,
231 * - it has space available.
233 * On success, return a "__rseq_percpu" encoded pointer to the pool
234 * item. This encoded pointer is meant to be passed to rseq_percpu_ptr()
235 * to be decoded to a valid address before being accessed.
237 * Return NULL (errno=ENOMEM) if there is not enough space left in the
238 * pool to allocate an item.
240 * This API is MT-safe.
242 void __rseq_percpu
*rseq_percpu_pool_set_malloc(struct rseq_percpu_pool_set
*pool_set
, size_t len
);
245 * rseq_percpu_pool_set_zmalloc: Allocated zero-initialized memory from a per-cpu pool set.
247 * Allocate memory for an item within the pool, and zero-initialize its
248 * memory on all CPUs. See rseq_percpu_pool_set_malloc for details.
250 * This API is MT-safe.
252 void __rseq_percpu
*rseq_percpu_pool_set_zmalloc(struct rseq_percpu_pool_set
*pool_set
, size_t len
);
258 #endif /* _RSEQ_PERCPU_ALLOC_H */
This page took 0.034539 seconds and 5 git commands to generate.