From cb475906285382fff5418190657e4f013a538e3e Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Fri, 8 Mar 2024 21:47:22 -0500 Subject: [PATCH] mempool: Receive mempool type (percpu/global) as attribute Also introduce a smp.c/h helper to get the number of possible cpus, copied from libside (MIT). Signed-off-by: Mathieu Desnoyers Change-Id: I4dfc8fa56ae29e5586eb029377abad62c93de6a9 --- include/rseq/mempool.h | 52 +++++--- src/Makefile.am | 2 +- src/rseq-mempool.c | 109 +++++++++++----- src/smp.c | 278 +++++++++++++++++++++++++++++++++++++++++ src/smp.h | 12 ++ tests/mempool_test.c | 17 ++- tests/param_test.c | 95 ++++++++++++-- 7 files changed, 499 insertions(+), 66 deletions(-) create mode 100644 src/smp.c create mode 100644 src/smp.h diff --git a/include/rseq/mempool.h b/include/rseq/mempool.h index 428661e..1813a16 100644 --- a/include/rseq/mempool.h +++ b/include/rseq/mempool.h @@ -35,7 +35,7 @@ extern "C" { * - rseq_percpu_ptr(). * - rseq_mempool_percpu_free(), */ -#define RSEQ_PERCPU_STRIDE (1U << 16) /* stride: 64kB */ +#define RSEQ_MEMPOOL_STRIDE (1U << 16) /* stride: 64kB */ /* * Tag pointers returned by: @@ -58,21 +58,20 @@ struct rseq_mempool; /* * rseq_mempool_create: Create a memory pool. * - * Create a per-cpu memory pool for items of size @item_len (rounded to - * next power of two). The reserved allocation size is @percpu_stride, and - * the maximum CPU value expected is (@max_nr_cpus - 1). A - * @percpu_stride of 0 uses the default RSEQ_PERCPU_STRIDE. + * Create a memory pool for items of size @item_len (rounded to + * next power of two). * * The @attr pointer used to specify the pool attributes. If NULL, use a * default attribute values. The @attr can be destroyed immediately * after rseq_mempool_create() returns. The caller keeps ownership - * of @attr. + * of @attr. Default attributes select a per-cpu mempool type. * * The argument @pool_name can be used to given a name to the pool for * debugging purposes. It can be NULL if no name is given. * * Returns a pointer to the created percpu pool. Return NULL on error, * with errno set accordingly: + * * EINVAL: Invalid argument. * ENOMEM: Not enough resources (memory or pool indexes) available to * allocate pool. @@ -84,8 +83,7 @@ struct rseq_mempool; * This API is MT-safe. */ struct rseq_mempool *rseq_mempool_create(const char *pool_name, - size_t item_len, size_t percpu_stride, int max_nr_cpus, - const struct rseq_mempool_attr *attr); + size_t item_len, const struct rseq_mempool_attr *attr); /* * rseq_mempool_destroy: Destroy a per-cpu memory pool. @@ -177,14 +175,14 @@ void *rseq_mempool_zmalloc(struct rseq_mempool *pool) * * The @stride optional argument to rseq_percpu_free() is a configurable * stride, which must match the stride received by pool creation. - * If the argument is not present, use the default RSEQ_PERCPU_STRIDE. + * If the argument is not present, use the default RSEQ_MEMPOOL_STRIDE. * * This API is MT-safe. */ -void librseq_mempool_percpu_free(void __rseq_percpu *ptr, size_t percpu_stride); +void librseq_mempool_percpu_free(void __rseq_percpu *ptr, size_t stride); #define rseq_mempool_percpu_free(_ptr, _stride...) \ - librseq_mempool_percpu_free(_ptr, RSEQ_PARAM_SELECT_ARG1(_, ##_stride, RSEQ_PERCPU_STRIDE)) + librseq_mempool_percpu_free(_ptr, RSEQ_PARAM_SELECT_ARG1(_, ##_stride, RSEQ_MEMPOOL_STRIDE)) /* * rseq_free: Free memory from a global pool. @@ -201,14 +199,14 @@ void librseq_mempool_percpu_free(void __rseq_percpu *ptr, size_t percpu_stride); * * The @stride optional argument to rseq_free() is a configurable * stride, which must match the stride received by pool creation. If - * the argument is not present, use the default RSEQ_PERCPU_STRIDE. + * the argument is not present, use the default RSEQ_MEMPOOL_STRIDE. * The stride is needed even for a global pool to know the mapping * address range. * * This API is MT-safe. */ #define rseq_mempool_free(_ptr, _stride...) \ - librseq_percpu_free((void __rseq_percpu *) _ptr, RSEQ_PARAM_SELECT_ARG1(_, ##_stride, RSEQ_PERCPU_STRIDE)) + librseq_percpu_free((void __rseq_percpu *) _ptr, RSEQ_PARAM_SELECT_ARG1(_, ##_stride, RSEQ_MEMPOOL_STRIDE)) /* * rseq_percpu_ptr: Offset a per-cpu pointer for a given CPU. @@ -226,7 +224,7 @@ void librseq_mempool_percpu_free(void __rseq_percpu *ptr, size_t percpu_stride); * for the returned pointer, but removes the __rseq_percpu annotation. * * The macro rseq_percpu_ptr() takes an optional @stride argument. If - * the argument is not present, use the default RSEQ_PERCPU_STRIDE. + * the argument is not present, use the default RSEQ_MEMPOOL_STRIDE. * This must match the stride used for pool creation. * * This API is MT-safe. @@ -234,7 +232,7 @@ void librseq_mempool_percpu_free(void __rseq_percpu *ptr, size_t percpu_stride); #define rseq_percpu_ptr(_ptr, _cpu, _stride...) \ ((__typeof__(*(_ptr)) *) ((uintptr_t) (_ptr) + \ ((unsigned int) (_cpu) * \ - (uintptr_t) RSEQ_PARAM_SELECT_ARG1(_, ##_stride, RSEQ_PERCPU_STRIDE)))) + (uintptr_t) RSEQ_PARAM_SELECT_ARG1(_, ##_stride, RSEQ_MEMPOOL_STRIDE)))) /* * rseq_mempool_set_create: Create a pool set. @@ -409,6 +407,30 @@ int rseq_mempool_attr_set_mmap(struct rseq_mempool_attr *attr, */ int rseq_mempool_attr_set_robust(struct rseq_mempool_attr *attr); +/* + * rseq_mempool_attr_set_percpu: Set pool type as percpu. + * + * A pool created with this type is a per-cpu memory pool. The reserved + * allocation size is @stride, and the maximum CPU value expected + * is (@max_nr_cpus - 1). A @stride of 0 uses the default + * RSEQ_MEMPOOL_STRIDE. + * + * Returns 0 on success, -1 with errno=EINVAL if arguments are invalid. + */ +int rseq_mempool_attr_set_percpu(struct rseq_mempool_attr *attr, + size_t stride, int max_nr_cpus); + +/* + * rseq_mempool_attr_set_global: Set pool type as global. + * + * A pool created with this type is a global memory pool. The reserved + * allocation size is @stride. A @stride of 0 uses the default + * RSEQ_MEMPOOL_STRIDE. + * + * Returns 0 on success, -1 with errno=EINVAL if arguments are invalid. + */ +int rseq_mempool_attr_set_global(struct rseq_mempool_attr *attr, size_t stride); + #ifdef __cplusplus } #endif diff --git a/src/Makefile.am b/src/Makefile.am index bb627a1..e313f27 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -4,7 +4,7 @@ lib_LTLIBRARIES = librseq.la librseq_la_SOURCES = \ - rseq.c rseq-mempool.c rseq-utils.h + rseq.c rseq-mempool.c rseq-utils.h smp.c smp.h librseq_la_LDFLAGS = -no-undefined -version-info $(RSEQ_LIBRARY_VERSION) librseq_la_LIBADD = $(DL_LIBS) diff --git a/src/rseq-mempool.c b/src/rseq-mempool.c index 8910dff..cc30d18 100644 --- a/src/rseq-mempool.c +++ b/src/rseq-mempool.c @@ -20,6 +20,7 @@ #endif #include "rseq-utils.h" +#include "smp.h" /* * rseq-mempool.c: rseq CPU-Local Storage (CLS) memory allocator. @@ -63,6 +64,11 @@ struct free_list_node { struct free_list_node *next; }; +enum mempool_type { + MEMPOOL_TYPE_PERCPU = 0, /* Default */ + MEMPOOL_TYPE_GLOBAL = 1, +}; + struct rseq_mempool_attr { bool mmap_set; void *(*mmap_func)(void *priv, size_t len); @@ -70,6 +76,10 @@ struct rseq_mempool_attr { void *mmap_priv; bool robust_set; + + enum mempool_type type; + size_t stride; + int max_nr_cpus; }; struct rseq_mempool_range; @@ -89,9 +99,7 @@ struct rseq_mempool { struct rseq_mempool_range *ranges; size_t item_len; - size_t percpu_stride; int item_order; - int max_nr_cpus; /* * The free list chains freed items on the CPU 0 address range. @@ -133,9 +141,9 @@ void rseq_percpu_zero_item(struct rseq_mempool *pool, uintptr_t item_offset) { int i; - for (i = 0; i < pool->max_nr_cpus; i++) { + for (i = 0; i < pool->attr.max_nr_cpus; i++) { char *p = __rseq_pool_percpu_ptr(pool, i, - item_offset, pool->percpu_stride); + item_offset, pool->attr.stride); memset(p, 0, pool->item_len); } } @@ -153,8 +161,8 @@ int rseq_mempool_range_init_numa(struct rseq_mempool *pool, struct rseq_mempool_ if (!numa_flags) return 0; page_len = rseq_get_page_len(); - nr_pages = pool->percpu_stride >> rseq_get_count_order_ulong(page_len); - for (cpu = 0; cpu < pool->max_nr_cpus; cpu++) { + nr_pages = pool->attr.stride >> rseq_get_count_order_ulong(page_len); + for (cpu = 0; cpu < pool->attr.max_nr_cpus; cpu++) { int status[MOVE_PAGES_BATCH_SIZE]; int nodes[MOVE_PAGES_BATCH_SIZE]; @@ -243,7 +251,7 @@ int create_alloc_bitmap(struct rseq_mempool *pool, struct rseq_mempool_range *ra { size_t count; - count = ((pool->percpu_stride >> pool->item_order) + BIT_PER_ULONG - 1) / BIT_PER_ULONG; + count = ((pool->attr.stride >> pool->item_order) + BIT_PER_ULONG - 1) / BIT_PER_ULONG; /* * Not being able to create the validation bitmap is an error @@ -285,8 +293,8 @@ void check_free_list(const struct rseq_mempool *pool) return; for (range = pool->ranges; range; range = range->next) { - total_item += pool->percpu_stride >> pool->item_order; - total_never_allocated += (pool->percpu_stride - range->next_unused) >> pool->item_order; + total_item += pool->attr.stride >> pool->item_order; + total_never_allocated += (pool->attr.stride - range->next_unused) >> pool->item_order; } max_list_traversal = total_item - total_never_allocated; @@ -335,7 +343,7 @@ void destroy_alloc_bitmap(struct rseq_mempool *pool, struct rseq_mempool_range * if (!bitmap) return; - count = ((pool->percpu_stride >> pool->item_order) + BIT_PER_ULONG - 1) / BIT_PER_ULONG; + count = ((pool->attr.stride >> pool->item_order) + BIT_PER_ULONG - 1) / BIT_PER_ULONG; /* Assert that all items in the pool were freed. */ for (size_t k = 0; k < count; ++k) @@ -357,7 +365,7 @@ int rseq_mempool_range_destroy(struct rseq_mempool *pool, destroy_alloc_bitmap(pool, range); /* range is a header located one page before the aligned mapping. */ return pool->attr.munmap_func(pool->attr.mmap_priv, range->header, - (pool->percpu_stride * pool->max_nr_cpus) + rseq_get_page_len()); + (pool->attr.stride * pool->attr.max_nr_cpus) + rseq_get_page_len()); } /* @@ -453,8 +461,8 @@ struct rseq_mempool_range *rseq_mempool_range_create(struct rseq_mempool *pool) page_size = rseq_get_page_len(); base = aligned_mmap_anonymous(pool, page_size, - pool->percpu_stride * pool->max_nr_cpus, - pool->percpu_stride, + pool->attr.stride * pool->attr.max_nr_cpus, + pool->attr.stride, &header, page_size); if (!base) return NULL; @@ -496,8 +504,7 @@ end: } struct rseq_mempool *rseq_mempool_create(const char *pool_name, - size_t item_len, size_t percpu_stride, int max_nr_cpus, - const struct rseq_mempool_attr *_attr) + size_t item_len, const struct rseq_mempool_attr *_attr) { struct rseq_mempool *pool; struct rseq_mempool_attr attr = {}; @@ -515,16 +522,6 @@ struct rseq_mempool *rseq_mempool_create(const char *pool_name, } item_len = 1UL << order; - if (!percpu_stride) - percpu_stride = RSEQ_PERCPU_STRIDE; /* Use default */ - - if (max_nr_cpus < 0 || item_len > percpu_stride || - percpu_stride < (size_t) rseq_get_page_len() || - !is_pow2(percpu_stride)) { - errno = EINVAL; - return NULL; - } - if (_attr) memcpy(&attr, _attr, sizeof(attr)); if (!attr.mmap_set) { @@ -533,14 +530,38 @@ struct rseq_mempool *rseq_mempool_create(const char *pool_name, attr.mmap_priv = NULL; } + switch (attr.type) { + case MEMPOOL_TYPE_PERCPU: + if (attr.max_nr_cpus < 0) { + errno = EINVAL; + return NULL; + } + if (attr.max_nr_cpus == 0) { + /* Auto-detect */ + attr.max_nr_cpus = get_possible_cpus_array_len(); + if (attr.max_nr_cpus == 0) { + errno = EINVAL; + return NULL; + } + } + break; + case MEMPOOL_TYPE_GLOBAL: + break; + } + if (!attr.stride) + attr.stride = RSEQ_MEMPOOL_STRIDE; /* Use default */ + if (item_len > attr.stride || attr.stride < (size_t) rseq_get_page_len() || + !is_pow2(attr.stride)) { + errno = EINVAL; + return NULL; + } + pool = calloc(1, sizeof(struct rseq_mempool)); if (!pool) return NULL; memcpy(&pool->attr, &attr, sizeof(attr)); pthread_mutex_init(&pool->lock, NULL); - pool->percpu_stride = percpu_stride; - pool->max_nr_cpus = max_nr_cpus; pool->item_len = item_len; pool->item_order = order; @@ -603,7 +624,7 @@ void __rseq_percpu *__rseq_percpu_malloc(struct rseq_mempool *pool, bool zeroed) addr = (void __rseq_percpu *) (pool->ranges->base + item_offset); goto end; } - if (pool->ranges->next_unused + pool->item_len > pool->percpu_stride) { + if (pool->ranges->next_unused + pool->item_len > pool->attr.stride) { errno = ENOMEM; addr = NULL; goto end; @@ -655,13 +676,13 @@ void clear_alloc_slot(struct rseq_mempool *pool, size_t item_offset) bitmap[k] &= ~mask; } -void librseq_mempool_percpu_free(void __rseq_percpu *_ptr, size_t percpu_stride) +void librseq_mempool_percpu_free(void __rseq_percpu *_ptr, size_t stride) { uintptr_t ptr = (uintptr_t) _ptr; - void *range_base = (void *) (ptr & (~(percpu_stride - 1))); + void *range_base = (void *) (ptr & (~(stride - 1))); struct rseq_mempool_range *range = (struct rseq_mempool_range *) (range_base - RANGE_HEADER_OFFSET); struct rseq_mempool *pool = range->pool; - uintptr_t item_offset = ptr & (percpu_stride - 1); + uintptr_t item_offset = ptr & (stride - 1); struct free_list_node *head, *item; pthread_mutex_lock(&pool->lock); @@ -810,3 +831,29 @@ int rseq_mempool_attr_set_robust(struct rseq_mempool_attr *attr) attr->robust_set = true; return 0; } + +int rseq_mempool_attr_set_percpu(struct rseq_mempool_attr *attr, + size_t stride, int max_nr_cpus) +{ + if (!attr) { + errno = EINVAL; + return -1; + } + attr->type = MEMPOOL_TYPE_PERCPU; + attr->stride = stride; + attr->max_nr_cpus = max_nr_cpus; + return 0; +} + +int rseq_mempool_attr_set_global(struct rseq_mempool_attr *attr, + size_t stride) +{ + if (!attr) { + errno = EINVAL; + return -1; + } + attr->type = MEMPOOL_TYPE_GLOBAL; + attr->stride = stride; + attr->max_nr_cpus = 1; + return 0; +} diff --git a/src/smp.c b/src/smp.c new file mode 100644 index 0000000..9788e93 --- /dev/null +++ b/src/smp.c @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (C) 2011-2012 Mathieu Desnoyers + * Copyright (C) 2019 Michael Jeanson + */ + +#define _LGPL_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "smp.h" + +#define __max(a,b) ((a)>(b)?(a):(b)) + +#define SIDE_CPUMASK_SIZE 4096 + +static int possible_cpus_array_len_cache; + +static +int _get_max_cpuid_from_sysfs(const char *path) +{ + long max_cpuid = -1; + + DIR *cpudir; + struct dirent *entry; + + assert(path); + + cpudir = opendir(path); + if (cpudir == NULL) + goto end; + + /* + * Iterate on all directories named "cpu" followed by an integer. + */ + while ((entry = readdir(cpudir))) { + if (entry->d_type == DT_DIR && + strncmp(entry->d_name, "cpu", 3) == 0) { + + char *endptr; + long cpu_id; + + cpu_id = strtol(entry->d_name + 3, &endptr, 10); + if ((cpu_id < LONG_MAX) && (endptr != entry->d_name + 3) + && (*endptr == '\0')) { + if (cpu_id > max_cpuid) + max_cpuid = cpu_id; + } + } + } + + if (closedir(cpudir)) + perror("closedir"); + + /* + * If the max CPU id is out of bound, set it to -1 so it results in a + * CPU num of 0. + */ + if (max_cpuid < 0 || max_cpuid > INT_MAX) + max_cpuid = -1; + +end: + return max_cpuid; +} + +/* + * Get the highest CPU id from sysfs. + * + * Iterate on all the folders in "/sys/devices/system/cpu" that start with + * "cpu" followed by an integer, keep the highest CPU id encountered during + * this iteration and add 1 to get a number of CPUs. + * + * Returns the highest CPU id, or -1 on error. + */ +static +int get_max_cpuid_from_sysfs(void) +{ + return _get_max_cpuid_from_sysfs("/sys/devices/system/cpu"); +} + +/* + * As a fallback to parsing the CPU mask in "/sys/devices/system/cpu/possible", + * iterate on all the folders in "/sys/devices/system/cpu" that start with + * "cpu" followed by an integer, keep the highest CPU id encountered during + * this iteration and add 1 to get a number of CPUs. + * + * Then get the value from sysconf(_SC_NPROCESSORS_CONF) as a fallback and + * return the highest one. + * + * On Linux, using the value from sysconf can be unreliable since the way it + * counts CPUs varies between C libraries and even between versions of the same + * library. If we used it directly, getcpu() could return a value greater than + * this sysconf, in which case the arrays indexed by processor would overflow. + * + * As another example, the MUSL libc implementation of the _SC_NPROCESSORS_CONF + * sysconf does not return the number of configured CPUs in the system but + * relies on the cpu affinity mask of the current task. + * + * Returns 0 or less on error. + */ +static +int get_num_possible_cpus_fallback(void) +{ + /* + * Get the sysconf value as a last resort. Keep the highest number. + */ + return __max(sysconf(_SC_NPROCESSORS_CONF), get_max_cpuid_from_sysfs() + 1); +} + +/* + * Get a CPU mask string from sysfs. + * + * buf: the buffer where the mask will be read. + * max_bytes: the maximum number of bytes to write in the buffer. + * path: file path to read the mask from. + * + * Returns the number of bytes read or -1 on error. + */ +static +int get_cpu_mask_from_sysfs(char *buf, size_t max_bytes, const char *path) +{ + ssize_t bytes_read = 0; + size_t total_bytes_read = 0; + int fd = -1, ret = -1; + + assert(path); + + if (buf == NULL) + goto end; + + fd = open(path, O_RDONLY); + if (fd < 0) + goto end; + + do { + bytes_read = read(fd, buf + total_bytes_read, + max_bytes - total_bytes_read); + + if (bytes_read < 0) { + if (errno == EINTR) { + continue; /* retry operation */ + } else { + goto end; + } + } + + total_bytes_read += bytes_read; + assert(total_bytes_read <= max_bytes); + } while (max_bytes > total_bytes_read && bytes_read > 0); + + /* + * Make sure the mask read is a null terminated string. + */ + if (total_bytes_read < max_bytes) + buf[total_bytes_read] = '\0'; + else + buf[max_bytes - 1] = '\0'; + + if (total_bytes_read > INT_MAX) + goto end; + ret = (int) total_bytes_read; +end: + if (fd >= 0 && close(fd) < 0) + perror("close"); + return ret; +} + +/* + * Get the CPU possible mask string from sysfs. + * + * buf: the buffer where the mask will be read. + * max_bytes: the maximum number of bytes to write in the buffer. + * + * Returns the number of bytes read or -1 on error. + */ +static +int get_possible_cpu_mask_from_sysfs(char *buf, size_t max_bytes) +{ + return get_cpu_mask_from_sysfs(buf, max_bytes, + "/sys/devices/system/cpu/possible"); +} + +/* + * Get the highest CPU id from a CPU mask. + * + * pmask: the mask to parse. + * len: the len of the mask excluding '\0'. + * + * Returns the highest CPU id from the mask or -1 on error. + */ +static +int get_max_cpuid_from_mask(const char *pmask, size_t len) +{ + ssize_t i; + unsigned long cpu_index; + char *endptr; + + /* We need at least one char to read */ + if (len < 1) + goto error; + + /* Start from the end to read the last CPU index. */ + for (i = len - 1; i > 0; i--) { + /* Break when we hit the first separator. */ + if ((pmask[i] == ',') || (pmask[i] == '-')) { + i++; + break; + } + } + + cpu_index = strtoul(&pmask[i], &endptr, 10); + + if ((&pmask[i] != endptr) && (cpu_index < INT_MAX)) + return (int) cpu_index; + +error: + return -1; +} + +static void update_possible_cpus_array_len_cache(void) +{ + char buf[SIDE_CPUMASK_SIZE]; + int ret; + + /* Get the possible cpu mask from sysfs, fallback to sysconf. */ + ret = get_possible_cpu_mask_from_sysfs((char *) &buf, SIDE_CPUMASK_SIZE); + if (ret <= 0) + goto fallback; + + /* Parse the possible cpu mask, on failure fallback to sysconf. */ + ret = get_max_cpuid_from_mask((char *) &buf, ret); + if (ret >= 0) { + /* Add 1 to convert from max cpuid to an array len. */ + ret++; + goto end; + } + +fallback: + /* Fallback to sysconf. */ + ret = get_num_possible_cpus_fallback(); + +end: + /* If all methods failed, don't store the value. */ + if (ret < 1) + return; + + possible_cpus_array_len_cache = ret; +} + +/* + * Returns the length of an array that could contain a per-CPU element for each + * possible CPU id for the lifetime of the process. + * + * We currently assume CPU ids are contiguous up the maximum CPU id. + * + * If the cache is not yet initialized, get the value from + * "/sys/devices/system/cpu/possible" or fallback to sysconf and cache it. + * + * If all methods fail, don't populate the cache and return 0. + */ +int get_possible_cpus_array_len(void) +{ + if (side_unlikely(!possible_cpus_array_len_cache)) + update_possible_cpus_array_len_cache(); + + return possible_cpus_array_len_cache; +} diff --git a/src/smp.h b/src/smp.h new file mode 100644 index 0000000..6f9240f --- /dev/null +++ b/src/smp.h @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright (C) 2011-2012 Mathieu Desnoyers + * Copyright (C) 2019 Michael Jeanson + */ + +#ifndef _RSEQ_SMP_H +#define _RSEQ_SMP_H + +int get_possible_cpus_array_len(void) __attribute__((visibility("hidden"))); + +#endif /* _RSEQ_SMP_H */ diff --git a/tests/mempool_test.c b/tests/mempool_test.c index 8911fde..6e19246 100644 --- a/tests/mempool_test.c +++ b/tests/mempool_test.c @@ -44,10 +44,10 @@ static void test_mempool_fill(size_t stride) ok(attr, "Create pool attribute"); ret = rseq_mempool_attr_set_robust(attr); ok(ret == 0, "Setting mempool robust attribute"); - + ret = rseq_mempool_attr_set_percpu(attr, stride, CPU_SETSIZE); + ok(ret == 0, "Setting mempool percpu type"); mempool = rseq_mempool_create("test_data", - sizeof(struct test_data), - stride, CPU_SETSIZE, attr); + sizeof(struct test_data), attr); ok(mempool, "Create mempool of size %zu", stride); rseq_mempool_attr_destroy(attr); @@ -167,14 +167,19 @@ static void run_robust_tests(void) { struct rseq_mempool_attr *attr; struct rseq_mempool *pool; + int ret; attr = rseq_mempool_attr_create(); + ok(attr, "Create mempool attributes"); + + ret = rseq_mempool_attr_set_robust(attr); + ok(ret == 0, "Setting mempool robust attribute"); - rseq_mempool_attr_set_robust(attr); + ret = rseq_mempool_attr_set_percpu(attr, RSEQ_MEMPOOL_STRIDE, 1); + ok(ret == 0, "Setting mempool percpu type"); pool = rseq_mempool_create("mempool-robust", - sizeof(void*), RSEQ_PERCPU_STRIDE, 1, - attr); + sizeof(void*), attr); rseq_mempool_attr_destroy(attr); diff --git a/tests/param_test.c b/tests/param_test.c index ac4f12c..62614c3 100644 --- a/tests/param_test.c +++ b/tests/param_test.c @@ -496,14 +496,25 @@ static void test_percpu_spinlock(void) struct spinlock_test_data __rseq_percpu *data; struct spinlock_thread_test_data thread_data[num_threads]; struct rseq_mempool *mempool; + struct rseq_mempool_attr *attr; + attr = rseq_mempool_attr_create(); + if (!attr) { + perror("rseq_mempool_attr_create"); + abort(); + } + ret = rseq_mempool_attr_set_percpu(attr, RSEQ_MEMPOOL_STRIDE, CPU_SETSIZE); + if (ret) { + perror("rseq_mempool_attr_set_percpu"); + abort(); + } mempool = rseq_mempool_create("spinlock_test_data", - sizeof(struct spinlock_test_data), - 0, CPU_SETSIZE, NULL); + sizeof(struct spinlock_test_data), attr); if (!mempool) { perror("rseq_mempool_create"); abort(); } + rseq_mempool_attr_destroy(attr); data = (struct spinlock_test_data __rseq_percpu *)rseq_mempool_percpu_zmalloc(mempool); if (!data) { perror("rseq_mempool_percpu_zmalloc"); @@ -592,14 +603,25 @@ static void test_percpu_inc(void) struct inc_test_data __rseq_percpu *data; struct inc_thread_test_data thread_data[num_threads]; struct rseq_mempool *mempool; + struct rseq_mempool_attr *attr; + attr = rseq_mempool_attr_create(); + if (!attr) { + perror("rseq_mempool_attr_create"); + abort(); + } + ret = rseq_mempool_attr_set_percpu(attr, RSEQ_MEMPOOL_STRIDE, CPU_SETSIZE); + if (ret) { + perror("rseq_mempool_attr_set_percpu"); + abort(); + } mempool = rseq_mempool_create("inc_test_data", - sizeof(struct inc_test_data), - 0, CPU_SETSIZE, NULL); + sizeof(struct inc_test_data), attr); if (!mempool) { perror("rseq_mempool_create"); abort(); } + rseq_mempool_attr_destroy(attr); data = (struct inc_test_data __rseq_percpu *)rseq_mempool_percpu_zmalloc(mempool); if (!data) { perror("rseq_mempool_percpu_zmalloc"); @@ -766,13 +788,25 @@ static void test_percpu_list(void) pthread_t test_threads[num_threads]; cpu_set_t allowed_cpus; struct rseq_mempool *mempool; + struct rseq_mempool_attr *attr; - mempool = rseq_mempool_create("percpu_list", sizeof(struct percpu_list), - 0, CPU_SETSIZE, NULL); + attr = rseq_mempool_attr_create(); + if (!attr) { + perror("rseq_mempool_attr_create"); + abort(); + } + ret = rseq_mempool_attr_set_percpu(attr, RSEQ_MEMPOOL_STRIDE, CPU_SETSIZE); + if (ret) { + perror("rseq_mempool_attr_set_percpu"); + abort(); + } + mempool = rseq_mempool_create("percpu_list", + sizeof(struct percpu_list), attr); if (!mempool) { perror("rseq_mempool_create"); abort(); } + rseq_mempool_attr_destroy(attr); list = (struct percpu_list __rseq_percpu *)rseq_mempool_percpu_zmalloc(mempool); if (!list) { perror("rseq_mempool_percpu_zmalloc"); @@ -977,13 +1011,25 @@ static void test_percpu_buffer(void) pthread_t test_threads[num_threads]; cpu_set_t allowed_cpus; struct rseq_mempool *mempool; + struct rseq_mempool_attr *attr; - mempool = rseq_mempool_create("percpu_buffer", sizeof(struct percpu_buffer), - 0, CPU_SETSIZE, NULL); + attr = rseq_mempool_attr_create(); + if (!attr) { + perror("rseq_mempool_attr_create"); + abort(); + } + ret = rseq_mempool_attr_set_percpu(attr, RSEQ_MEMPOOL_STRIDE, CPU_SETSIZE); + if (ret) { + perror("rseq_mempool_attr_set_percpu"); + abort(); + } + mempool = rseq_mempool_create("percpu_buffer", + sizeof(struct percpu_buffer), attr); if (!mempool) { perror("rseq_mempool_create"); abort(); } + rseq_mempool_attr_destroy(attr); buffer = (struct percpu_buffer __rseq_percpu *)rseq_mempool_percpu_zmalloc(mempool); if (!buffer) { perror("rseq_mempool_percpu_zmalloc"); @@ -1217,14 +1263,25 @@ static void test_percpu_memcpy_buffer(void) pthread_t test_threads[num_threads]; cpu_set_t allowed_cpus; struct rseq_mempool *mempool; + struct rseq_mempool_attr *attr; + attr = rseq_mempool_attr_create(); + if (!attr) { + perror("rseq_mempool_attr_create"); + abort(); + } + ret = rseq_mempool_attr_set_percpu(attr, RSEQ_MEMPOOL_STRIDE, CPU_SETSIZE); + if (ret) { + perror("rseq_mempool_attr_set_percpu"); + abort(); + } mempool = rseq_mempool_create("percpu_memcpy_buffer", - sizeof(struct percpu_memcpy_buffer), - 0, CPU_SETSIZE, NULL); + sizeof(struct percpu_memcpy_buffer), attr); if (!mempool) { perror("rseq_mempool_create"); abort(); } + rseq_mempool_attr_destroy(attr); buffer = (struct percpu_memcpy_buffer __rseq_percpu *)rseq_mempool_percpu_zmalloc(mempool); if (!buffer) { perror("rseq_mempool_percpu_zmalloc"); @@ -1390,7 +1447,7 @@ void *test_membarrier_worker_thread(void *arg) ret = rseq_load_add_load_load_add_store__ptr(RSEQ_MO_RELAXED, RSEQ_PERCPU, (intptr_t *) &args->percpu_list_ptr, - (RSEQ_PERCPU_STRIDE * cpu) + offsetof(struct percpu_list, head), + (RSEQ_MEMPOOL_STRIDE * cpu) + offsetof(struct percpu_list, head), 1, cpu); } while (rseq_unlikely(ret)); } @@ -1463,13 +1520,25 @@ void *test_membarrier_manager_thread(void *arg) struct rseq_mempool *mempool; int ret; long long total_count = 0; + struct rseq_mempool_attr *attr; - mempool = rseq_mempool_create("percpu_list", sizeof(struct percpu_list), - 0, CPU_SETSIZE, NULL); + attr = rseq_mempool_attr_create(); + if (!attr) { + perror("rseq_mempool_attr_create"); + abort(); + } + ret = rseq_mempool_attr_set_percpu(attr, RSEQ_MEMPOOL_STRIDE, CPU_SETSIZE); + if (ret) { + perror("rseq_mempool_attr_set_percpu"); + abort(); + } + mempool = rseq_mempool_create("percpu_list", + sizeof(struct percpu_list), attr); if (!mempool) { perror("rseq_mempool_create"); abort(); } + rseq_mempool_attr_destroy(attr); args->mempool = mempool; if (rseq_register_current_thread()) { -- 2.34.1