include/rseq/percpu-alloc.h

   1 /* SPDX-License-Identifier: MIT */
   2 /* SPDX-FileCopyrightText: 2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
   3
   4 #ifndef _RSEQ_PERCPU_ALLOC_H
   5 #define _RSEQ_PERCPU_ALLOC_H
   6
   7 #include <stddef.h>
   8 #include <sys/types.h>
   9 #include <sys/mman.h>
  10
  11 /*
  12  * rseq/percpu-alloc.h: rseq CPU-Local Storage (CLS) memory allocator.
  13  *
  14  * The rseq per-CPU memory allocator allows the application the request
  15  * memory pools of CPU-Local memory each of containing objects of a
  16  * given size (rounded to next power of 2), reserving a given virtual
  17  * address size per CPU, for a given maximum number of CPUs.
  18  *
  19  * The per-CPU memory allocator is analogous to TLS (Thread-Local
  20  * Storage) memory: TLS is Thread-Local Storage, whereas the per-CPU
  21  * memory allocator provides CPU-Local Storage.
  22  */
  23
  24 #ifdef __cplusplus
  25 extern "C" {
  26 #endif
  27
  28 /*
  29  * Tag pointers returned by:
  30  * - rseq_percpu_malloc(),
  31  * - rseq_percpu_zmalloc(),
  32  * - rseq_percpu_pool_set_malloc(),
  33  * - rseq_percpu_pool_set_zmalloc().
  34  *
  35  * and passed as parameter to:
  36  * - rseq_percpu_ptr(),
  37  * - rseq_percpu_free().
  38  *
  39  * with __rseq_percpu for use by static analyzers.
  40  */
  41 #define __rseq_percpu
  42
  43 struct rseq_pool_attr;
  44 struct rseq_percpu_pool;
  45
  46 /*
  47  * Create a robust pool.  This enables the following runtime checks:
  48  *
  49  *   - Check for double free of pointers.
  50  *
  51  *   - Check that all items were freed when destroying the pool, i.e. no memory
  52  *     leak.
  53  *
  54  *  There is a marginal runtime overhead on malloc/free operations.
  55  *
  56  *  The memory overhead is (pool->percpu_len / pool->item_len) / CHAR_BIT
  57  *  bytes, over the lifetime of the pool.
  58  */
  59 #define RSEQ_POOL_ROBUST    (1 << 0)
  60
  61 /*
  62  * rseq_percpu_pool_create: Create a per-cpu memory pool.
  63  *
  64  * Create a per-cpu memory pool for items of size @item_len (rounded to
  65  * next power of two). The reserved allocation size is @percpu_len, and
  66  * the maximum CPU value expected is (@max_nr_cpus - 1).
  67  *
  68  * The @attr pointer used to specify the pool attributes. If NULL, use a
  69  * default attribute values. The @attr can be destroyed immediately
  70  * after rseq_percpu_pool_create() returns. The caller keeps ownership
  71  * of @attr.
  72  *
  73  * The argument @pool_name can be used to given a name to the pool for
  74  * debugging purposes. It can be NULL if no name is given.
  75  *
  76  * Returns a pointer to the created percpu pool. Return NULL on error,
  77  * with errno set accordingly:
  78  *   EINVAL: Invalid argument.
  79  *   ENOMEM: Not enough resources (memory or pool indexes) available to
  80  *           allocate pool.
  81  *
  82  * In addition, if the attr mmap callback fails, NULL is returned and
  83  * errno is propagated from the callback. The default callback can
  84  * return errno=ENOMEM.
  85  *
  86  * This API is MT-safe.
  87  */
  88 struct rseq_percpu_pool *rseq_percpu_pool_create(const char *pool_name,
  89                 size_t item_len, size_t percpu_len, int max_nr_cpus,
  90                 const struct rseq_pool_attr *attr);
  91
  92 /*
  93  * rseq_percpu_pool_destroy: Destroy a per-cpu memory pool.
  94  *
  95  * Destroy a per-cpu memory pool, unmapping its memory and removing the
  96  * pool entry from the global index. No pointers allocated from the
  97  * pool should be used when it is destroyed. This includes rseq_percpu_ptr().
  98  *
  99  * Argument @pool is a pointer to the per-cpu pool to destroy.
 100  *
 101  * Return values: 0 on success, -1 on error, with errno set accordingly:
 102  *   ENOENT: Trying to free a pool which was not allocated.
 103  *
 104  * If the munmap_func callback fails, -1 is returned and errno is
 105  * propagated from the callback. The default callback can return
 106  * errno=EINVAL.
 107  *
 108  * This API is MT-safe.
 109  */
 110 int rseq_percpu_pool_destroy(struct rseq_percpu_pool *pool);
 111
 112 /*
 113  * rseq_percpu_malloc: Allocate memory from a per-cpu pool.
 114  *
 115  * Allocate an item from a per-cpu @pool. The allocation will reserve
 116  * an item of the size specified by @item_len (rounded to next power of
 117  * two) at pool creation. This effectively reserves space for this item
 118  * on all CPUs.
 119  *
 120  * On success, return a "__rseq_percpu" encoded pointer to the pool
 121  * item. This encoded pointer is meant to be passed to rseq_percpu_ptr()
 122  * to be decoded to a valid address before being accessed.
 123  *
 124  * Return NULL (errno=ENOMEM) if there is not enough space left in the
 125  * pool to allocate an item.
 126  *
 127  * This API is MT-safe.
 128  */
 129 void __rseq_percpu *rseq_percpu_malloc(struct rseq_percpu_pool *pool);
 130
 131 /*
 132  * rseq_percpu_zmalloc: Allocated zero-initialized memory from a per-cpu pool.
 133  *
 134  * Allocate memory for an item within the pool, and zero-initialize its
 135  * memory on all CPUs. See rseq_percpu_malloc for details.
 136  *
 137  * This API is MT-safe.
 138  */
 139 void __rseq_percpu *rseq_percpu_zmalloc(struct rseq_percpu_pool *pool);
 140
 141 /*
 142  * rseq_percpu_free: Free memory from a per-cpu pool.
 143  *
 144  * Free an item pointed to by @ptr from its per-cpu pool.
 145  *
 146  * The @ptr argument is a __rseq_percpu encoded pointer returned by
 147  * either:
 148  *
 149  * - rseq_percpu_malloc(),
 150  * - rseq_percpu_zmalloc(),
 151  * - rseq_percpu_pool_set_malloc(),
 152  * - rseq_percpu_pool_set_zmalloc().
 153  *
 154  * This API is MT-safe.
 155  */
 156 void rseq_percpu_free(void __rseq_percpu *ptr);
 157
 158 /*
 159  * rseq_percpu_ptr: Decode a per-cpu pointer.
 160  *
 161  * Decode a per-cpu pointer @ptr to get the associated pointer for the
 162  * given @cpu. The @ptr argument is a __rseq_percpu encoded pointer
 163  * returned by either:
 164  *
 165  * - rseq_percpu_malloc(),
 166  * - rseq_percpu_zmalloc(),
 167  * - rseq_percpu_pool_set_malloc(),
 168  * - rseq_percpu_pool_set_zmalloc().
 169  *
 170  * The __rseq_percpu pointer can be decoded with rseq_percpu_ptr() even
 171  * after it has been freed, as long as its associated pool has not been
 172  * destroyed. However, memory pointed to by the decoded pointer should
 173  * not be accessed after the __rseq_percpu pointer has been freed.
 174  *
 175  * The macro rseq_percpu_ptr() preserves the type of the @ptr parameter
 176  * for the returned pointer, but removes the __rseq_percpu annotation.
 177  *
 178  * This API is MT-safe.
 179  */
 180 void *__rseq_percpu_ptr(void __rseq_percpu *ptr, int cpu);
 181 #define rseq_percpu_ptr(ptr, cpu)       ((__typeof__(*(ptr)) *) __rseq_percpu_ptr(ptr, cpu))
 182
 183 /*
 184  * rseq_percpu_pool_set_create: Create a pool set.
 185  *
 186  * Create a set of pools. Its purpose is to offer a memory allocator API
 187  * for variable-length items (e.g. variable length strings). When
 188  * created, the pool set has no pool. Pools can be created and added to
 189  * the set. One common approach would be to create pools for each
 190  * relevant power of two allocation size useful for the application.
 191  * Only one pool can be added to the pool set for each power of two
 192  * allocation size.
 193  *
 194  * Returns a pool set pointer on success, else returns NULL with
 195  * errno=ENOMEM (out of memory).
 196  *
 197  * This API is MT-safe.
 198  */
 199 struct rseq_percpu_pool_set *rseq_percpu_pool_set_create(void);
 200
 201 /*
 202  * rseq_percpu_pool_set_destroy: Destroy a pool set.
 203  *
 204  * Destroy a pool set and its associated resources. The pools that were
 205  * added to the pool set are destroyed as well.
 206  *
 207  * Returns 0 on success, -1 on failure (or partial failure), with errno
 208  * set by rseq_percpu_pool_destroy(). Using a pool set after destroy
 209  * failure is undefined.
 210  *
 211  * This API is MT-safe.
 212  */
 213 int rseq_percpu_pool_set_destroy(struct rseq_percpu_pool_set *pool_set);
 214
 215 /*
 216  * rseq_percpu_pool_set_add_pool: Add a pool to a pool set.
 217  *
 218  * Add a @pool to the @pool_set. On success, its ownership is handed
 219  * over to the pool set, so the caller should not destroy it explicitly.
 220  * Only one pool can be added to the pool set for each power of two
 221  * allocation size.
 222  *
 223  * Returns 0 on success, -1 on error with the following errno:
 224  * - EBUSY: A pool already exists in the pool set for this power of two
 225  *          allocation size.
 226  *
 227  * This API is MT-safe.
 228  */
 229 int rseq_percpu_pool_set_add_pool(struct rseq_percpu_pool_set *pool_set,
 230                 struct rseq_percpu_pool *pool);
 231
 232 /*
 233  * rseq_percpu_pool_set_malloc: Allocate memory from a per-cpu pool set.
 234  *
 235  * Allocate an item from a per-cpu @pool. The allocation will reserve
 236  * an item of the size specified by @len (rounded to next power of
 237  * two). This effectively reserves space for this item on all CPUs.
 238  *
 239  * The space reservation will search for the smallest pool within
 240  * @pool_set which respects the following conditions:
 241  *
 242  * - it has an item size large enough to fit @len,
 243  * - it has space available.
 244  *
 245  * On success, return a "__rseq_percpu" encoded pointer to the pool
 246  * item. This encoded pointer is meant to be passed to rseq_percpu_ptr()
 247  * to be decoded to a valid address before being accessed.
 248  *
 249  * Return NULL (errno=ENOMEM) if there is not enough space left in the
 250  * pool to allocate an item.
 251  *
 252  * This API is MT-safe.
 253  */
 254 void __rseq_percpu *rseq_percpu_pool_set_malloc(struct rseq_percpu_pool_set *pool_set, size_t len);
 255
 256 /*
 257  * rseq_percpu_pool_set_zmalloc: Allocated zero-initialized memory from a per-cpu pool set.
 258  *
 259  * Allocate memory for an item within the pool, and zero-initialize its
 260  * memory on all CPUs. See rseq_percpu_pool_set_malloc for details.
 261  *
 262  * This API is MT-safe.
 263  */
 264 void __rseq_percpu *rseq_percpu_pool_set_zmalloc(struct rseq_percpu_pool_set *pool_set, size_t len);
 265
 266 /*
 267  * rseq_percpu_pool_init_numa: Move pages to the NUMA node associated to their CPU topology.
 268  *
 269  * For pages allocated within @pool, invoke move_pages(2) with the given
 270  * @numa_flags to move the pages to the NUMA node associated to their
 271  * CPU topology.
 272  *
 273  * Argument @numa_flags are passed to move_pages(2). The expected flags are:
 274  *   MPOL_MF_MOVE:     move process-private pages to cpu-specific numa nodes.
 275  *   MPOL_MF_MOVE_ALL: move shared pages to cpu-specific numa nodes
 276  *                     (requires CAP_SYS_NICE).
 277  *
 278  * Returns 0 on success, else return -1 with errno set by move_pages(2).
 279  */
 280 int rseq_percpu_pool_init_numa(struct rseq_percpu_pool *pool, int numa_flags);
 281
 282 /*
 283  * rseq_pool_attr_create: Create a pool attribute structure.
 284  */
 285 struct rseq_pool_attr *rseq_pool_attr_create(void);
 286
 287 /*
 288  * rseq_pool_attr_destroy: Destroy a pool attribute structure.
 289  */
 290 void rseq_pool_attr_destroy(struct rseq_pool_attr *attr);
 291
 292 /*
 293  * rseq_pool_attr_set_mmap: Set pool attribute structure mmap functions.
 294  *
 295  * The @mmap_func callback used to map the memory for the pool.
 296  *
 297  * The @munmap_func callback used to unmap the memory when the pool
 298  * is destroyed.
 299  *
 300  * The @mmap_priv argument is a private data pointer passed to both
 301  * @mmap_func and @munmap_func callbacks.
 302  *
 303  * Returns 0 on success, -1 with errno=EINVAL if arguments are invalid.
 304  */
 305 int rseq_pool_attr_set_mmap(struct rseq_pool_attr *attr,
 306                 void *(*mmap_func)(void *priv, size_t len),
 307                 int (*munmap_func)(void *priv, void *ptr, size_t len),
 308                 void *mmap_priv);
 309
 310 /*
 311  * rseq_pool_attr_set_robust: Set pool robust attribute.
 312  *
 313  * The robust pool attribute enables runtime validation of the pool:
 314  *
 315  *   - Check for double-free of pointers.
 316  *
 317  *   - Detect memory leaks on pool destruction.
 318   *
 319  *   - Detect free-list corruption on pool destruction.
 320  *
 321  * There is a marginal runtime overhead on malloc/free operations.
 322  *
 323  * The memory overhead is (pool->percpu_len / pool->item_len) / CHAR_BIT
 324  * bytes, over the lifetime of the pool.
 325  *
 326  * Returns 0 on success, -1 with errno=EINVAL if arguments are invalid.
 327  */
 328 int rseq_pool_attr_set_robust(struct rseq_pool_attr *attr);
 329
 330 #ifdef __cplusplus
 331 }
 332 #endif
 333
 334 #endif /* _RSEQ_PERCPU_ALLOC_H */