include/rseq/percpu-alloc.h

   1 /* SPDX-License-Identifier: MIT */
   2 /* SPDX-FileCopyrightText: 2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
   3
   4 #ifndef _RSEQ_PERCPU_ALLOC_H
   5 #define _RSEQ_PERCPU_ALLOC_H
   6
   7 #include <stddef.h>
   8 #include <sys/types.h>
   9 #include <sys/mman.h>
  10
  11 /*
  12  * rseq/percpu-alloc.h: rseq CPU-Local Storage (CLS) memory allocator.
  13  *
  14  * The rseq per-CPU memory allocator allows the application the request
  15  * memory pools of CPU-Local memory each of containing objects of a
  16  * given size (rounded to next power of 2), reserving a given virtual
  17  * address size per CPU, for a given maximum number of CPUs.
  18  *
  19  * The per-CPU memory allocator is analogous to TLS (Thread-Local
  20  * Storage) memory: TLS is Thread-Local Storage, whereas the per-CPU
  21  * memory allocator provides CPU-Local Storage.
  22  */
  23
  24 #ifdef __cplusplus
  25 extern "C" {
  26 #endif
  27
  28 /*
  29  * Tag pointers returned by:
  30  * - rseq_percpu_malloc(),
  31  * - rseq_percpu_zmalloc(),
  32  * - rseq_percpu_pool_set_malloc(),
  33  * - rseq_percpu_pool_set_zmalloc().
  34  *
  35  * and passed as parameter to:
  36  * - rseq_percpu_ptr(),
  37  * - rseq_percpu_free().
  38  *
  39  * with __rseq_percpu for use by static analyzers.
  40  */
  41 #define __rseq_percpu
  42
  43 struct rseq_mmap_attr;
  44 struct rseq_percpu_pool;
  45
  46 /*
  47  * rseq_percpu_pool_create: Create a per-cpu memory pool.
  48  *
  49  * Create a per-cpu memory pool for items of size @item_len (rounded to
  50  * next power of two). The reserved allocation size is @percpu_len, and
  51  * the maximum CPU value expected is (@max_nr_cpus - 1).
  52  *
  53  * The @mmap_attr pointer used to specify the memory allocator callbacks
  54  * to use to manage the memory for the pool. If NULL, use a default
  55  * internal implementation. The @mmap_attr can be destroyed immediately
  56  * after rseq_percpu_pool_create() returns. The caller keeps ownership
  57  * of @mmap_attr.
  58  *
  59  * Argument @flags is currently expected to be 0. This is for future
  60  * extensions.
  61  *
  62  * Returns a pointer to the created percpu pool. Return NULL on error,
  63  * with errno set accordingly:
  64  *   EINVAL: Invalid argument.
  65  *   ENOMEM: Not enough resources (memory or pool indexes) available to
  66  *           allocate pool.
  67  *
  68  * In addition, if the mmap_attr mmap callback fails, NULL is returned
  69  * and errno is propagated from the callback. The default callback can
  70  * return errno=ENOMEM.
  71  *
  72  * This API is MT-safe.
  73  */
  74 struct rseq_percpu_pool *rseq_percpu_pool_create(size_t item_len,
  75                 size_t percpu_len, int max_nr_cpus,
  76                 const struct rseq_mmap_attr *mmap_attr,
  77                 int flags);
  78
  79 /*
  80  * rseq_percpu_pool_destroy: Destroy a per-cpu memory pool.
  81  *
  82  * Destroy a per-cpu memory pool, unmapping its memory and removing the
  83  * pool entry from the global index. No pointers allocated from the
  84  * pool should be used when it is destroyed. This includes rseq_percpu_ptr().
  85  *
  86  * Argument @pool is a pointer to the per-cpu pool to destroy.
  87  *
  88  * Return values: 0 on success, -1 on error, with errno set accordingly:
  89  *   ENOENT: Trying to free a pool which was not allocated.
  90  *
  91  * If the munmap_func callback fails, -1 is returned and errno is
  92  * propagated from the callback. The default callback can return
  93  * errno=EINVAL.
  94  *
  95  * This API is MT-safe.
  96  */
  97 int rseq_percpu_pool_destroy(struct rseq_percpu_pool *pool);
  98
  99 /*
 100  * rseq_percpu_malloc: Allocate memory from a per-cpu pool.
 101  *
 102  * Allocate an item from a per-cpu @pool. The allocation will reserve
 103  * an item of the size specified by @item_len (rounded to next power of
 104  * two) at pool creation. This effectively reserves space for this item
 105  * on all CPUs.
 106  *
 107  * On success, return a "__rseq_percpu" encoded pointer to the pool
 108  * item. This encoded pointer is meant to be passed to rseq_percpu_ptr()
 109  * to be decoded to a valid address before being accessed.
 110  *
 111  * Return NULL (errno=ENOMEM) if there is not enough space left in the
 112  * pool to allocate an item.
 113  *
 114  * This API is MT-safe.
 115  */
 116 void __rseq_percpu *rseq_percpu_malloc(struct rseq_percpu_pool *pool);
 117
 118 /*
 119  * rseq_percpu_zmalloc: Allocated zero-initialized memory from a per-cpu pool.
 120  *
 121  * Allocate memory for an item within the pool, and zero-initialize its
 122  * memory on all CPUs. See rseq_percpu_malloc for details.
 123  *
 124  * This API is MT-safe.
 125  */
 126 void __rseq_percpu *rseq_percpu_zmalloc(struct rseq_percpu_pool *pool);
 127
 128 /*
 129  * rseq_percpu_free: Free memory from a per-cpu pool.
 130  *
 131  * Free an item pointed to by @ptr from its per-cpu pool.
 132  *
 133  * The @ptr argument is a __rseq_percpu encoded pointer returned by
 134  * either:
 135  *
 136  * - rseq_percpu_malloc(),
 137  * - rseq_percpu_zmalloc(),
 138  * - rseq_percpu_pool_set_malloc(),
 139  * - rseq_percpu_pool_set_zmalloc().
 140  *
 141  * This API is MT-safe.
 142  */
 143 void rseq_percpu_free(void __rseq_percpu *ptr);
 144
 145 /*
 146  * rseq_percpu_ptr: Decode a per-cpu pointer.
 147  *
 148  * Decode a per-cpu pointer @ptr to get the associated pointer for the
 149  * given @cpu. The @ptr argument is a __rseq_percpu encoded pointer
 150  * returned by either:
 151  *
 152  * - rseq_percpu_malloc(),
 153  * - rseq_percpu_zmalloc(),
 154  * - rseq_percpu_pool_set_malloc(),
 155  * - rseq_percpu_pool_set_zmalloc().
 156  *
 157  * The __rseq_percpu pointer can be decoded with rseq_percpu_ptr() even
 158  * after it has been freed, as long as its associated pool has not been
 159  * destroyed. However, memory pointed to by the decoded pointer should
 160  * not be accessed after the __rseq_percpu pointer has been freed.
 161  *
 162  * The macro rseq_percpu_ptr() preserves the type of the @ptr parameter
 163  * for the returned pointer, but removes the __rseq_percpu annotation.
 164  *
 165  * This API is MT-safe.
 166  */
 167 void *__rseq_percpu_ptr(void __rseq_percpu *ptr, int cpu);
 168 #define rseq_percpu_ptr(ptr, cpu)       ((__typeof__(*(ptr)) *) __rseq_percpu_ptr(ptr, cpu))
 169
 170 /*
 171  * rseq_percpu_pool_cpu_offset: Return the offset from encoded to decoded percpu pointer.
 172  *
 173  * Calculate the offset from any __rseq_percpu pointer allocated from
 174  * the pool to its associated per-cpu data for @cpu.
 175  *
 176  * This API is MT-safe.
 177  */
 178 ptrdiff_t rseq_percpu_pool_ptr_offset(struct rseq_percpu_pool *pool, int cpu);
 179
 180 /*
 181  * rseq_percpu_pool_set_create: Create a pool set.
 182  *
 183  * Create a set of pools. Its purpose is to offer a memory allocator API
 184  * for variable-length items (e.g. variable length strings). When
 185  * created, the pool set has no pool. Pools can be created and added to
 186  * the set. One common approach would be to create pools for each
 187  * relevant power of two allocation size useful for the application.
 188  * Only one pool can be added to the pool set for each power of two
 189  * allocation size.
 190  *
 191  * Returns a pool set pointer on success, else returns NULL with
 192  * errno=ENOMEM (out of memory).
 193  *
 194  * This API is MT-safe.
 195  */
 196 struct rseq_percpu_pool_set *rseq_percpu_pool_set_create(void);
 197
 198 /*
 199  * rseq_percpu_pool_set_destroy: Destroy a pool set.
 200  *
 201  * Destroy a pool set and its associated resources. The pools that were
 202  * added to the pool set are destroyed as well.
 203  *
 204  * Returns 0 on success, -1 on failure (or partial failure), with errno
 205  * set by rseq_percpu_pool_destroy(). Using a pool set after destroy
 206  * failure is undefined.
 207  *
 208  * This API is MT-safe.
 209  */
 210 int rseq_percpu_pool_set_destroy(struct rseq_percpu_pool_set *pool_set);
 211
 212 /*
 213  * rseq_percpu_pool_set_add_pool: Add a pool to a pool set.
 214  *
 215  * Add a @pool to the @pool_set. On success, its ownership is handed
 216  * over to the pool set, so the caller should not destroy it explicitly.
 217  * Only one pool can be added to the pool set for each power of two
 218  * allocation size.
 219  *
 220  * Returns 0 on success, -1 on error with the following errno:
 221  * - EBUSY: A pool already exists in the pool set for this power of two
 222  *          allocation size.
 223  *
 224  * This API is MT-safe.
 225  */
 226 int rseq_percpu_pool_set_add_pool(struct rseq_percpu_pool_set *pool_set,
 227                 struct rseq_percpu_pool *pool);
 228
 229 /*
 230  * rseq_percpu_pool_set_malloc: Allocate memory from a per-cpu pool set.
 231  *
 232  * Allocate an item from a per-cpu @pool. The allocation will reserve
 233  * an item of the size specified by @len (rounded to next power of
 234  * two). This effectively reserves space for this item on all CPUs.
 235  *
 236  * The space reservation will search for the smallest pool within
 237  * @pool_set which respects the following conditions:
 238  *
 239  * - it has an item size large enough to fit @len,
 240  * - it has space available.
 241  *
 242  * On success, return a "__rseq_percpu" encoded pointer to the pool
 243  * item. This encoded pointer is meant to be passed to rseq_percpu_ptr()
 244  * to be decoded to a valid address before being accessed.
 245  *
 246  * Return NULL (errno=ENOMEM) if there is not enough space left in the
 247  * pool to allocate an item.
 248  *
 249  * This API is MT-safe.
 250  */
 251 void __rseq_percpu *rseq_percpu_pool_set_malloc(struct rseq_percpu_pool_set *pool_set, size_t len);
 252
 253 /*
 254  * rseq_percpu_pool_set_zmalloc: Allocated zero-initialized memory from a per-cpu pool set.
 255  *
 256  * Allocate memory for an item within the pool, and zero-initialize its
 257  * memory on all CPUs. See rseq_percpu_pool_set_malloc for details.
 258  *
 259  * This API is MT-safe.
 260  */
 261 void __rseq_percpu *rseq_percpu_pool_set_zmalloc(struct rseq_percpu_pool_set *pool_set, size_t len);
 262
 263 /*
 264  * rseq_percpu_pool_init_numa: Move pages to the NUMA node associated to their CPU topology.
 265  *
 266  * For pages allocated within @pool, invoke move_pages(2) with the given
 267  * @numa_flags to move the pages to the NUMA node associated to their
 268  * CPU topology.
 269  *
 270  * Argument @numa_flags are passed to move_pages(2). The expected flags are:
 271  *   MPOL_MF_MOVE:     move process-private pages to cpu-specific numa nodes.
 272  *   MPOL_MF_MOVE_ALL: move shared pages to cpu-specific numa nodes
 273  *                     (requires CAP_SYS_NICE).
 274  *
 275  * Returns 0 on success, else return -1 with errno set by move_pages(2).
 276  */
 277 int rseq_percpu_pool_init_numa(struct rseq_percpu_pool *pool, int numa_flags);
 278
 279 /*
 280  * rseq_mmap_attr_create: Create a mmap attribute structure.
 281  *
 282  * The @mmap_func callback used to map the memory for the pool.
 283  *
 284  * The @munmap_func callback used to unmap the memory when the pool
 285  * is destroyed.
 286  *
 287  * The @mmap_priv argument is a private data pointer passed to both
 288  * @mmap_func and @munmap_func callbacks.
 289  */
 290 struct rseq_mmap_attr *rseq_mmap_attr_create(void *(*mmap_func)(void *priv, size_t len),
 291                 int (*munmap_func)(void *priv, void *ptr, size_t len),
 292                 void *mmap_priv);
 293
 294 /*
 295  * rseq_mmap_attr_destroy: Destroy a mmap attribute structure.
 296  */
 297 void rseq_mmap_attr_destroy(struct rseq_mmap_attr *attr);
 298
 299 #ifdef __cplusplus
 300 }
 301 #endif
 302
 303 #endif /* _RSEQ_PERCPU_ALLOC_H */