include/rseq/mempool.h

   1 /* SPDX-License-Identifier: MIT */
   2 /* SPDX-FileCopyrightText: 2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
   3
   4 #ifndef _RSEQ_MEMPOOL_H
   5 #define _RSEQ_MEMPOOL_H
   6
   7 #include <rseq/compiler.h>
   8 #include <stddef.h>
   9 #include <sys/types.h>
  10 #include <sys/mman.h>
  11
  12 /*
  13  * rseq/mempool.h: rseq CPU-Local Storage (CLS) memory allocator.
  14  *
  15  * The rseq per-CPU memory allocator allows the application the request
  16  * memory pools of CPU-Local memory each of containing objects of a
  17  * given size (rounded to next power of 2), reserving a given virtual
  18  * address size per CPU, for a given maximum number of CPUs.
  19  *
  20  * The per-CPU memory allocator is analogous to TLS (Thread-Local
  21  * Storage) memory: TLS is Thread-Local Storage, whereas the per-CPU
  22  * memory allocator provides CPU-Local Storage.
  23  */
  24
  25 #ifdef __cplusplus
  26 extern "C" {
  27 #endif
  28
  29 /*
  30  * The percpu offset stride can be overridden by the user code.
  31  * The stride *must* match for all objects belonging to a given pool
  32  * between arguments to:
  33  *
  34  * - rseq_mempool_create(),
  35  * - rseq_percpu_ptr().
  36  * - rseq_percpu_free(),
  37  */
  38 #if RSEQ_BITS_PER_LONG == 64
  39 # define RSEQ_PERCPU_STRIDE     (1U << 24)      /* 64-bit stride: 16MB */
  40 #else
  41 # define RSEQ_PERCPU_STRIDE     (1U << 16)      /* 32-bit stride: 64kB */
  42 #endif
  43
  44 /*
  45  * Tag pointers returned by:
  46  * - rseq_percpu_malloc(),
  47  * - rseq_percpu_zmalloc(),
  48  * - rseq_percpu_pool_set_malloc(),
  49  * - rseq_percpu_pool_set_zmalloc().
  50  *
  51  * and passed as parameter to:
  52  * - rseq_percpu_ptr(),
  53  * - rseq_percpu_free().
  54  *
  55  * with __rseq_percpu for use by static analyzers.
  56  */
  57 #define __rseq_percpu
  58
  59 struct rseq_mempool_attr;
  60 struct rseq_mempool;
  61
  62 /*
  63  * rseq_percpu_pool_create: Create a per-cpu memory pool.
  64  *
  65  * Create a per-cpu memory pool for items of size @item_len (rounded to
  66  * next power of two). The reserved allocation size is @percpu_stride, and
  67  * the maximum CPU value expected is (@max_nr_cpus - 1). A
  68  * @percpu_stride of 0 uses the default RSEQ_PERCPU_STRIDE.
  69  *
  70  * The @attr pointer used to specify the pool attributes. If NULL, use a
  71  * default attribute values. The @attr can be destroyed immediately
  72  * after rseq_percpu_pool_create() returns. The caller keeps ownership
  73  * of @attr.
  74  *
  75  * The argument @pool_name can be used to given a name to the pool for
  76  * debugging purposes. It can be NULL if no name is given.
  77  *
  78  * Returns a pointer to the created percpu pool. Return NULL on error,
  79  * with errno set accordingly:
  80  *   EINVAL: Invalid argument.
  81  *   ENOMEM: Not enough resources (memory or pool indexes) available to
  82  *           allocate pool.
  83  *
  84  * In addition, if the attr mmap callback fails, NULL is returned and
  85  * errno is propagated from the callback. The default callback can
  86  * return errno=ENOMEM.
  87  *
  88  * This API is MT-safe.
  89  */
  90 struct rseq_mempool *rseq_mempool_create(const char *pool_name,
  91                 size_t item_len, size_t percpu_stride, int max_nr_cpus,
  92                 const struct rseq_mempool_attr *attr);
  93
  94 /*
  95  * rseq_mempool_destroy: Destroy a per-cpu memory pool.
  96  *
  97  * Destroy a per-cpu memory pool, unmapping its memory and removing the
  98  * pool entry from the global index. No pointers allocated from the
  99  * pool should be used when it is destroyed. This includes rseq_percpu_ptr().
 100  *
 101  * Argument @pool is a pointer to the per-cpu pool to destroy.
 102  *
 103  * Return values: 0 on success, -1 on error, with errno set accordingly:
 104  *   ENOENT: Trying to free a pool which was not allocated.
 105  *
 106  * If the munmap_func callback fails, -1 is returned and errno is
 107  * propagated from the callback. The default callback can return
 108  * errno=EINVAL.
 109  *
 110  * This API is MT-safe.
 111  */
 112 int rseq_mempool_destroy(struct rseq_mempool *pool);
 113
 114 /*
 115  * rseq_mempool_percpu_malloc: Allocate memory from a per-cpu pool.
 116  *
 117  * Allocate an item from a per-cpu @pool. The allocation will reserve
 118  * an item of the size specified by @item_len (rounded to next power of
 119  * two) at pool creation. This effectively reserves space for this item
 120  * on all CPUs.
 121  *
 122  * On success, return a "__rseq_percpu" encoded pointer to the pool
 123  * item. This encoded pointer is meant to be passed to rseq_percpu_ptr()
 124  * to be decoded to a valid address before being accessed.
 125  *
 126  * Return NULL (errno=ENOMEM) if there is not enough space left in the
 127  * pool to allocate an item.
 128  *
 129  * This API is MT-safe.
 130  */
 131 void __rseq_percpu *rseq_mempool_percpu_malloc(struct rseq_mempool *pool);
 132
 133 /*
 134  * rseq_mempool_percpu_zmalloc: Allocated zero-initialized memory from a per-cpu pool.
 135  *
 136  * Allocate memory for an item within the pool, and zero-initialize its
 137  * memory on all CPUs. See rseq_mempool_percpu_malloc for details.
 138  *
 139  * This API is MT-safe.
 140  */
 141 void __rseq_percpu *rseq_mempool_percpu_zmalloc(struct rseq_mempool *pool);
 142
 143 /*
 144  * rseq_mempool_malloc: Allocate memory from a global pool.
 145  *
 146  * Wrapper to allocate memory from a global pool, which can be
 147  * used directly without per-cpu indexing. Would normally be used
 148  * with pools created with max_nr_cpus=1.
 149  */
 150 static inline
 151 void *rseq_mempool_malloc(struct rseq_mempool *pool)
 152 {
 153         return (void *) rseq_mempool_percpu_malloc(pool);
 154 }
 155
 156 /*
 157  * rseq_mempool_zmalloc: Allocate zero-initialized memory from a global pool.
 158  *
 159  * Wrapper to allocate memory from a global pool, which can be
 160  * used directly without per-cpu indexing. Would normally be used
 161  * with pools created with max_nr_cpus=1.
 162  */
 163 static inline
 164 void *rseq_mempool_zmalloc(struct rseq_mempool *pool)
 165 {
 166         return (void *) rseq_mempool_percpu_zmalloc(pool);
 167 }
 168
 169 /*
 170  * rseq_mempool_percpu_free: Free memory from a per-cpu pool.
 171  *
 172  * Free an item pointed to by @ptr from its per-cpu pool.
 173  *
 174  * The @ptr argument is a __rseq_percpu encoded pointer returned by
 175  * either:
 176  *
 177  * - rseq_mempool_percpu_malloc(),
 178  * - rseq_mempool_percpu_zmalloc(),
 179  * - rseq_mempool_set_percpu_malloc(),
 180  * - rseq_mempool_set_percpu_zmalloc().
 181  *
 182  * The @stride optional argument to rseq_percpu_free() is a configurable
 183  * stride, which must match the stride received by pool creation.
 184  * If the argument is not present, use the default RSEQ_PERCPU_STRIDE.
 185  *
 186  * This API is MT-safe.
 187  */
 188 void librseq_mempool_percpu_free(void __rseq_percpu *ptr, size_t percpu_stride);
 189
 190 #define rseq_mempool_percpu_free(_ptr, _stride...)              \
 191         librseq_mempool_percpu_free(_ptr, RSEQ_PARAM_SELECT_ARG1(_, ##_stride, RSEQ_PERCPU_STRIDE))
 192
 193 /*
 194  * rseq_free: Free memory from a global pool.
 195  *
 196  * Free an item pointed to by @ptr from its global pool. Would normally
 197  * be used with pools created with max_nr_cpus=1.
 198  *
 199  * The @ptr argument is a pointer returned by either:
 200  *
 201  * - rseq_mempool_malloc(),
 202  * - rseq_mempool_zmalloc(),
 203  * - rseq_mempool_set_malloc(),
 204  * - rseq_mempool_set_zmalloc().
 205  *
 206  * The @stride optional argument to rseq_free() is a configurable
 207  * stride, which must match the stride received by pool creation. If
 208  * the argument is not present, use the default RSEQ_PERCPU_STRIDE.
 209  * The stride is needed even for a global pool to know the mapping
 210  * address range.
 211  *
 212  * This API is MT-safe.
 213  */
 214 #define rseq_mempool_free(_ptr, _stride...)             \
 215         librseq_percpu_free((void __rseq_percpu *) _ptr, RSEQ_PARAM_SELECT_ARG1(_, ##_stride, RSEQ_PERCPU_STRIDE))
 216
 217 /*
 218  * rseq_percpu_ptr: Offset a per-cpu pointer for a given CPU.
 219  *
 220  * Offset a per-cpu pointer @ptr to get the associated pointer for the
 221  * given @cpu. The @ptr argument is a __rseq_percpu pointer returned by
 222  * either:
 223  *
 224  * - rseq_mempool_percpu_malloc(),
 225  * - rseq_mempool_percpu_zmalloc(),
 226  * - rseq_mempool_set_percpu_malloc(),
 227  * - rseq_mempool_set_percpu_zmalloc().
 228  *
 229  * The macro rseq_percpu_ptr() preserves the type of the @ptr parameter
 230  * for the returned pointer, but removes the __rseq_percpu annotation.
 231  *
 232  * The macro rseq_percpu_ptr() takes an optional @stride argument. If
 233  * the argument is not present, use the default RSEQ_PERCPU_STRIDE.
 234  * This must match the stride used for pool creation.
 235  *
 236  * This API is MT-safe.
 237  */
 238 #define rseq_percpu_ptr(_ptr, _cpu, _stride...)         \
 239         ((__typeof__(*(_ptr)) *) ((uintptr_t) (_ptr) +  \
 240                 ((unsigned int) (_cpu) *                \
 241                         (uintptr_t) RSEQ_PARAM_SELECT_ARG1(_, ##_stride, RSEQ_PERCPU_STRIDE))))
 242
 243 /*
 244  * rseq_mempool_set_create: Create a pool set.
 245  *
 246  * Create a set of pools. Its purpose is to offer a memory allocator API
 247  * for variable-length items (e.g. variable length strings). When
 248  * created, the pool set has no pool. Pools can be created and added to
 249  * the set. One common approach would be to create pools for each
 250  * relevant power of two allocation size useful for the application.
 251  * Only one pool can be added to the pool set for each power of two
 252  * allocation size.
 253  *
 254  * Returns a pool set pointer on success, else returns NULL with
 255  * errno=ENOMEM (out of memory).
 256  *
 257  * This API is MT-safe.
 258  */
 259 struct rseq_mempool_set *rseq_mempool_set_create(void);
 260
 261 /*
 262  * rseq_mempool_set_destroy: Destroy a pool set.
 263  *
 264  * Destroy a pool set and its associated resources. The pools that were
 265  * added to the pool set are destroyed as well.
 266  *
 267  * Returns 0 on success, -1 on failure (or partial failure), with errno
 268  * set by rseq_percpu_pool_destroy(). Using a pool set after destroy
 269  * failure is undefined.
 270  *
 271  * This API is MT-safe.
 272  */
 273 int rseq_mempool_set_destroy(struct rseq_mempool_set *pool_set);
 274
 275 /*
 276  * rseq_mempool_set_add_pool: Add a pool to a pool set.
 277  *
 278  * Add a @pool to the @pool_set. On success, its ownership is handed
 279  * over to the pool set, so the caller should not destroy it explicitly.
 280  * Only one pool can be added to the pool set for each power of two
 281  * allocation size.
 282  *
 283  * Returns 0 on success, -1 on error with the following errno:
 284  * - EBUSY: A pool already exists in the pool set for this power of two
 285  *          allocation size.
 286  *
 287  * This API is MT-safe.
 288  */
 289 int rseq_mempool_set_add_pool(struct rseq_mempool_set *pool_set,
 290                 struct rseq_mempool *pool);
 291
 292 /*
 293  * rseq_percpu_mempool_set_malloc: Allocate memory from a per-cpu pool set.
 294  *
 295  * Allocate an item from a per-cpu @pool. The allocation will reserve
 296  * an item of the size specified by @len (rounded to next power of
 297  * two). This effectively reserves space for this item on all CPUs.
 298  *
 299  * The space reservation will search for the smallest pool within
 300  * @pool_set which respects the following conditions:
 301  *
 302  * - it has an item size large enough to fit @len,
 303  * - it has space available.
 304  *
 305  * On success, return a "__rseq_percpu" encoded pointer to the pool
 306  * item. This encoded pointer is meant to be passed to rseq_percpu_ptr()
 307  * to be decoded to a valid address before being accessed.
 308  *
 309  * Return NULL (errno=ENOMEM) if there is not enough space left in the
 310  * pool to allocate an item.
 311  *
 312  * This API is MT-safe.
 313  */
 314 void __rseq_percpu *rseq_mempool_set_percpu_malloc(struct rseq_mempool_set *pool_set, size_t len);
 315
 316 /*
 317  * rseq_percpu_mempool_set_zmalloc: Allocated zero-initialized memory from a per-cpu pool set.
 318  *
 319  * Allocate memory for an item within the pool, and zero-initialize its
 320  * memory on all CPUs. See rseq_percpu_mempool_set_malloc for details.
 321  *
 322  * This API is MT-safe.
 323  */
 324 void __rseq_percpu *rseq_mempool_set_percpu_zmalloc(struct rseq_mempool_set *pool_set, size_t len);
 325
 326 /*
 327  * rseq_mempool_set_malloc: Allocate memory from a global pool set.
 328  *
 329  * Wrapper to allocate memory from a global pool, which can be
 330  * used directly without per-cpu indexing. Would normally be used
 331  * with pools created with max_nr_cpus=1.
 332  */
 333 static inline
 334 void *rseq_mempool_set_malloc(struct rseq_mempool_set *pool_set, size_t len)
 335 {
 336         return (void *) rseq_mempool_set_percpu_malloc(pool_set, len);
 337 }
 338
 339 /*
 340  * rseq_mempool_set_zmalloc: Allocate zero-initialized memory from a global pool set.
 341  *
 342  * Wrapper to allocate memory from a global pool, which can be
 343  * used directly without per-cpu indexing. Would normally be used
 344  * with pools created with max_nr_cpus=1.
 345  */
 346 static inline
 347 void *rseq_mempool_set_zmalloc(struct rseq_mempool_set *pool_set, size_t len)
 348 {
 349         return (void *) rseq_mempool_set_percpu_zmalloc(pool_set, len);
 350 }
 351
 352 /*
 353  * rseq_mempool_init_numa: Move pages to the NUMA node associated to their CPU topology.
 354  *
 355  * For pages allocated within @pool, invoke move_pages(2) with the given
 356  * @numa_flags to move the pages to the NUMA node associated to their
 357  * CPU topology.
 358  *
 359  * Argument @numa_flags are passed to move_pages(2). The expected flags are:
 360  *   MPOL_MF_MOVE:     move process-private pages to cpu-specific numa nodes.
 361  *   MPOL_MF_MOVE_ALL: move shared pages to cpu-specific numa nodes
 362  *                     (requires CAP_SYS_NICE).
 363  *
 364  * Returns 0 on success, else return -1 with errno set by move_pages(2).
 365  */
 366 int rseq_mempool_init_numa(struct rseq_mempool *pool, int numa_flags);
 367
 368 /*
 369  * rseq_mempool_attr_create: Create a pool attribute structure.
 370  */
 371 struct rseq_mempool_attr *rseq_mempool_attr_create(void);
 372
 373 /*
 374  * rseq_mempool_attr_destroy: Destroy a pool attribute structure.
 375  */
 376 void rseq_mempool_attr_destroy(struct rseq_mempool_attr *attr);
 377
 378 /*
 379  * rseq_mempool_attr_set_mmap: Set pool attribute structure mmap functions.
 380  *
 381  * The @mmap_func callback used to map the memory for the pool.
 382  *
 383  * The @munmap_func callback used to unmap the memory when the pool
 384  * is destroyed.
 385  *
 386  * The @mmap_priv argument is a private data pointer passed to both
 387  * @mmap_func and @munmap_func callbacks.
 388  *
 389  * Returns 0 on success, -1 with errno=EINVAL if arguments are invalid.
 390  */
 391 int rseq_mempool_attr_set_mmap(struct rseq_mempool_attr *attr,
 392                 void *(*mmap_func)(void *priv, size_t len),
 393                 int (*munmap_func)(void *priv, void *ptr, size_t len),
 394                 void *mmap_priv);
 395
 396 /*
 397  * rseq_mempool_attr_set_robust: Set pool robust attribute.
 398  *
 399  * The robust pool attribute enables runtime validation of the pool:
 400  *
 401  *   - Check for double-free of pointers.
 402  *
 403  *   - Detect memory leaks on pool destruction.
 404   *
 405  *   - Detect free-list corruption on pool destruction.
 406  *
 407  * There is a marginal runtime overhead on malloc/free operations.
 408  *
 409  * The memory overhead is (pool->percpu_len / pool->item_len) / CHAR_BIT
 410  * bytes, over the lifetime of the pool.
 411  *
 412  * Returns 0 on success, -1 with errno=EINVAL if arguments are invalid.
 413  */
 414 int rseq_mempool_attr_set_robust(struct rseq_mempool_attr *attr);
 415
 416 #ifdef __cplusplus
 417 }
 418 #endif
 419
 420 #endif /* _RSEQ_MEMPOOL_H */