include/rseq/percpu-alloc.h

   1 /* SPDX-License-Identifier: MIT */
   2 /* SPDX-FileCopyrightText: 2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
   3
   4 #ifndef _RSEQ_PERCPU_ALLOC_H
   5 #define _RSEQ_PERCPU_ALLOC_H
   6
   7 #include <stddef.h>
   8 #include <sys/types.h>
   9 #include <sys/mman.h>
  10
  11 /*
  12  * rseq/percpu-alloc.h: rseq CPU-Local Storage (CLS) memory allocator.
  13  *
  14  * The rseq per-CPU memory allocator allows the application the request
  15  * memory pools of CPU-Local memory each of containing objects of a
  16  * given size (rounded to next power of 2), reserving a given virtual
  17  * address size per CPU, for a given maximum number of CPUs.
  18  *
  19  * The per-CPU memory allocator is analogous to TLS (Thread-Local
  20  * Storage) memory: TLS is Thread-Local Storage, whereas the per-CPU
  21  * memory allocator provides CPU-Local Storage.
  22  */
  23
  24 #ifdef __cplusplus
  25 extern "C" {
  26 #endif
  27
  28
  29 /*
  30  * Tag pointers returned by:
  31  * - rseq_percpu_malloc(),
  32  * - rseq_percpu_zmalloc(),
  33  * - rseq_percpu_pool_set_malloc(),
  34  * - rseq_percpu_pool_set_zmalloc().
  35  *
  36  * and passed as parameter to:
  37  * - rseq_percpu_ptr(),
  38  * - rseq_percpu_free().
  39  *
  40  * with __rseq_percpu for use by static analyzers.
  41  */
  42 #define __rseq_percpu
  43
  44 struct rseq_percpu_pool;
  45
  46 /*
  47  * rseq_percpu_pool_create: Create a per-cpu memory pool.
  48  *
  49  * Create a per-cpu memory pool for items of size @item_len (rounded to
  50  * next power of two). The reserved allocation size is @percpu_len, and
  51  * the maximum CPU value expected is (@max_nr_cpus - 1).
  52  *
  53  * Arguments @mmap_prot, @mmap_flags, @mmap_fd, @mmap_offset are passed
  54  * as arguments to mmap(2) when allocating the memory area holding the
  55  * percpu pool.
  56  *
  57  * Argument @numa_flags are passed to move_pages(2). The expected flags
  58  * are:
  59  *   0:                do not move pages to specific numa nodes
  60  *                     (use for e.g. mm_cid indexing).
  61  *   MPOL_MF_MOVE:     move process-private pages to cpu-specific numa nodes.
  62  *   MPOL_MF_MOVE_ALL: move shared pages to cpu-specific numa nodes
  63  *                     (requires CAP_SYS_NICE).
  64  *
  65  * Returns a pointer to the created percpu pool. Return NULL on error,
  66  * with errno set accordingly:
  67  *   EINVAL: Invalid argument.
  68  *   ENOMEM: Not enough resources (memory or pool indexes) available to
  69  *           allocate pool.
  70  *
  71  * In addition, if mmap(2) fails, NULL is returned and errno is
  72  * propagated from mmap(2).
  73  *
  74  * This API is MT-safe.
  75  */
  76 struct rseq_percpu_pool *rseq_percpu_pool_create(size_t item_len,
  77                 size_t percpu_len, int max_nr_cpus,
  78                 int mmap_prot, int mmap_flags, int mmap_fd, off_t mmap_offset,
  79                 int numa_flags);
  80
  81 /*
  82  * rseq_percpu_pool_destroy: Destroy a per-cpu memory pool.
  83  *
  84  * Destroy a per-cpu memory pool, unmapping its memory and removing the
  85  * pool entry from the global index. No pointers allocated from the
  86  * pool should be used when it is destroyed. This includes rseq_percpu_ptr().
  87  *
  88  * Argument @pool is a pointer to the per-cpu pool to destroy.
  89  *
  90  * Return values: 0 on success, -1 on error, with errno set accordingly:
  91  *   ENOENT: Trying to free a pool which was not allocated.
  92  *
  93  * If munmap(2) fails, -1 is returned and errno is propagated from
  94  * munmap(2).
  95  *
  96  * This API is MT-safe.
  97  */
  98 int rseq_percpu_pool_destroy(struct rseq_percpu_pool *pool);
  99
 100 /*
 101  * rseq_percpu_malloc: Allocate memory from a per-cpu pool.
 102  *
 103  * Allocate an item from a per-cpu @pool. The allocation will reserve
 104  * an item of the size specified by @item_len (rounded to next power of
 105  * two) at pool creation. This effectively reserves space for this item
 106  * on all CPUs.
 107  *
 108  * On success, return a "__rseq_percpu" encoded pointer to the pool
 109  * item. This encoded pointer is meant to be passed to rseq_percpu_ptr()
 110  * to be decoded to a valid address before being accessed.
 111  *
 112  * Return NULL (errno=ENOMEM) if there is not enough space left in the
 113  * pool to allocate an item.
 114  *
 115  * This API is MT-safe.
 116  */
 117 void __rseq_percpu *rseq_percpu_malloc(struct rseq_percpu_pool *pool);
 118
 119 /*
 120  * rseq_percpu_zmalloc: Allocated zero-initialized memory from a per-cpu pool.
 121  *
 122  * Allocate memory for an item within the pool, and zero-initialize its
 123  * memory on all CPUs. See rseq_percpu_malloc for details.
 124  *
 125  * This API is MT-safe.
 126  */
 127 void __rseq_percpu *rseq_percpu_zmalloc(struct rseq_percpu_pool *pool);
 128
 129 /*
 130  * rseq_percpu_free: Free memory from a per-cpu pool.
 131  *
 132  * Free an item pointed to by @ptr from its per-cpu pool.
 133  *
 134  * The @ptr argument is a __rseq_percpu encoded pointer returned by
 135  * either:
 136  *
 137  * - rseq_percpu_malloc(),
 138  * - rseq_percpu_zmalloc(),
 139  * - rseq_percpu_pool_set_malloc(),
 140  * - rseq_percpu_pool_set_zmalloc().
 141  *
 142  * This API is MT-safe.
 143  */
 144 void rseq_percpu_free(void __rseq_percpu *ptr);
 145
 146 /*
 147  * rseq_percpu_ptr: Decode a per-cpu pointer.
 148  *
 149  * Decode a per-cpu pointer @ptr to get the associated pointer for the
 150  * given @cpu. The @ptr argument is a __rseq_percpu encoded pointer
 151  * returned by either:
 152  *
 153  * - rseq_percpu_malloc(),
 154  * - rseq_percpu_zmalloc(),
 155  * - rseq_percpu_pool_set_malloc(),
 156  * - rseq_percpu_pool_set_zmalloc().
 157  *
 158  * The __rseq_percpu pointer can be decoded with rseq_percpu_ptr() even
 159  * after it has been freed, as long as its associated pool has not been
 160  * destroyed. However, memory pointed to by the decoded pointer should
 161  * not be accessed after the __rseq_percpu pointer has been freed.
 162  *
 163  * The macro rseq_percpu_ptr() preserves the type of the @ptr parameter
 164  * for the returned pointer, but removes the __rseq_percpu annotation.
 165  *
 166  * This API is MT-safe.
 167  */
 168 void *__rseq_percpu_ptr(void __rseq_percpu *ptr, int cpu);
 169 #define rseq_percpu_ptr(ptr, cpu)       ((__typeof__(*(ptr)) *) __rseq_percpu_ptr(ptr, cpu))
 170
 171 /*
 172  * rseq_percpu_pool_set_create: Create a pool set.
 173  *
 174  * Create a set of pools. Its purpose is to offer a memory allocator API
 175  * for variable-length items (e.g. variable length strings). When
 176  * created, the pool set has no pool. Pools can be created and added to
 177  * the set. One common approach would be to create pools for each
 178  * relevant power of two allocation size useful for the application.
 179  * Only one pool can be added to the pool set for each power of two
 180  * allocation size.
 181  *
 182  * Returns a pool set pointer on success, else returns NULL with
 183  * errno=ENOMEM (out of memory).
 184  *
 185  * This API is MT-safe.
 186  */
 187 struct rseq_percpu_pool_set *rseq_percpu_pool_set_create(void);
 188
 189 /*
 190  * rseq_percpu_pool_set_destroy: Destroy a pool set.
 191  *
 192  * Destroy a pool set and its associated resources. The pools that were
 193  * added to the pool set are destroyed as well.
 194  *
 195  * Returns 0 on success, -1 on failure (or partial failure), with errno
 196  * set by rseq_percpu_pool_destroy(). Using a pool set after destroy
 197  * failure is undefined.
 198  *
 199  * This API is MT-safe.
 200  */
 201 int rseq_percpu_pool_set_destroy(struct rseq_percpu_pool_set *pool_set);
 202
 203 /*
 204  * rseq_percpu_pool_set_add_pool: Add a pool to a pool set.
 205  *
 206  * Add a @pool to the @pool_set. On success, its ownership is handed
 207  * over to the pool set, so the caller should not destroy it explicitly.
 208  * Only one pool can be added to the pool set for each power of two
 209  * allocation size.
 210  *
 211  * Returns 0 on success, -1 on error with the following errno:
 212  * - EBUSY: A pool already exists in the pool set for this power of two
 213  *          allocation size.
 214  *
 215  * This API is MT-safe.
 216  */
 217 int rseq_percpu_pool_set_add_pool(struct rseq_percpu_pool_set *pool_set,
 218                 struct rseq_percpu_pool *pool);
 219
 220 /*
 221  * rseq_percpu_pool_set_malloc: Allocate memory from a per-cpu pool set.
 222  *
 223  * Allocate an item from a per-cpu @pool. The allocation will reserve
 224  * an item of the size specified by @len (rounded to next power of
 225  * two). This effectively reserves space for this item on all CPUs.
 226  *
 227  * The space reservation will search for the smallest pool within
 228  * @pool_set which respects the following conditions:
 229  *
 230  * - it has an item size large enough to fit @len,
 231  * - it has space available.
 232  *
 233  * On success, return a "__rseq_percpu" encoded pointer to the pool
 234  * item. This encoded pointer is meant to be passed to rseq_percpu_ptr()
 235  * to be decoded to a valid address before being accessed.
 236  *
 237  * Return NULL (errno=ENOMEM) if there is not enough space left in the
 238  * pool to allocate an item.
 239  *
 240  * This API is MT-safe.
 241  */
 242 void __rseq_percpu *rseq_percpu_pool_set_malloc(struct rseq_percpu_pool_set *pool_set, size_t len);
 243
 244 /*
 245  * rseq_percpu_pool_set_zmalloc: Allocated zero-initialized memory from a per-cpu pool set.
 246  *
 247  * Allocate memory for an item within the pool, and zero-initialize its
 248  * memory on all CPUs. See rseq_percpu_pool_set_malloc for details.
 249  *
 250  * This API is MT-safe.
 251  */
 252 void __rseq_percpu *rseq_percpu_pool_set_zmalloc(struct rseq_percpu_pool_set *pool_set, size_t len);
 253
 254 #ifdef __cplusplus
 255 }
 256 #endif
 257
 258 #endif /* _RSEQ_PERCPU_ALLOC_H */