#endif
#include "rseq-utils.h"
+#include <rseq/rseq.h>
/*
* rseq-mempool.c: rseq CPU-Local Storage (CLS) memory allocator.
struct free_list_node *next;
};
+enum mempool_type {
+ MEMPOOL_TYPE_GLOBAL = 0, /* Default */
+ MEMPOOL_TYPE_PERCPU = 1,
+};
+
struct rseq_mempool_attr {
bool mmap_set;
void *(*mmap_func)(void *priv, size_t len);
int (*munmap_func)(void *priv, void *ptr, size_t len);
void *mmap_priv;
+ bool init_set;
+ int (*init_func)(void *priv, void *addr, size_t len, int cpu);
+ void *init_priv;
+
bool robust_set;
+
+ enum mempool_type type;
+ size_t stride;
+ int max_nr_cpus;
};
struct rseq_mempool_range;
struct rseq_mempool_range *ranges;
size_t item_len;
- size_t percpu_stride;
int item_order;
- int max_nr_cpus;
/*
* The free list chains freed items on the CPU 0 address range.
};
static
-void *__rseq_pool_percpu_ptr(struct rseq_mempool *pool, int cpu,
+void *__rseq_pool_range_percpu_ptr(struct rseq_mempool_range *range, int cpu,
uintptr_t item_offset, size_t stride)
{
- /* TODO: Implement multi-ranges support. */
- return pool->ranges->base + (stride * cpu) + item_offset;
+ return range->base + (stride * cpu) + item_offset;
}
static
-void rseq_percpu_zero_item(struct rseq_mempool *pool, uintptr_t item_offset)
+void rseq_percpu_zero_item(struct rseq_mempool *pool,
+ struct rseq_mempool_range *range, uintptr_t item_offset)
{
int i;
- for (i = 0; i < pool->max_nr_cpus; i++) {
- char *p = __rseq_pool_percpu_ptr(pool, i,
- item_offset, pool->percpu_stride);
+ for (i = 0; i < pool->attr.max_nr_cpus; i++) {
+ char *p = __rseq_pool_range_percpu_ptr(range, i,
+ item_offset, pool->attr.stride);
memset(p, 0, pool->item_len);
}
}
-//TODO: this will need to be reimplemented for ranges,
-//which cannot use __rseq_pool_percpu_ptr.
-#if 0 //#ifdef HAVE_LIBNUMA
-static
-int rseq_mempool_range_init_numa(struct rseq_mempool *pool, struct rseq_mempool_range *range, int numa_flags)
+#ifdef HAVE_LIBNUMA
+int rseq_mempool_range_init_numa(void *addr, size_t len, int cpu, int numa_flags)
{
unsigned long nr_pages, page_len;
+ int status[MOVE_PAGES_BATCH_SIZE];
+ int nodes[MOVE_PAGES_BATCH_SIZE];
+ void *pages[MOVE_PAGES_BATCH_SIZE];
long ret;
- int cpu;
- if (!numa_flags)
- return 0;
+ if (!numa_flags) {
+ errno = EINVAL;
+ return -1;
+ }
page_len = rseq_get_page_len();
- nr_pages = pool->percpu_stride >> rseq_get_count_order_ulong(page_len);
- for (cpu = 0; cpu < pool->max_nr_cpus; cpu++) {
+ nr_pages = len >> rseq_get_count_order_ulong(page_len);
- int status[MOVE_PAGES_BATCH_SIZE];
- int nodes[MOVE_PAGES_BATCH_SIZE];
- void *pages[MOVE_PAGES_BATCH_SIZE];
+ nodes[0] = numa_node_of_cpu(cpu);
+ if (nodes[0] < 0)
+ return -1;
- nodes[0] = numa_node_of_cpu(cpu);
- for (size_t k = 1; k < RSEQ_ARRAY_SIZE(nodes); ++k) {
- nodes[k] = nodes[0];
- }
+ for (size_t k = 1; k < RSEQ_ARRAY_SIZE(nodes); ++k) {
+ nodes[k] = nodes[0];
+ }
- for (unsigned long page = 0; page < nr_pages;) {
+ for (unsigned long page = 0; page < nr_pages;) {
- size_t max_k = RSEQ_ARRAY_SIZE(pages);
- size_t left = nr_pages - page;
+ size_t max_k = RSEQ_ARRAY_SIZE(pages);
+ size_t left = nr_pages - page;
- if (left < max_k) {
- max_k = left;
- }
+ if (left < max_k) {
+ max_k = left;
+ }
- for (size_t k = 0; k < max_k; ++k, ++page) {
- pages[k] = __rseq_pool_percpu_ptr(pool, cpu, page * page_len);
- status[k] = -EPERM;
- }
+ for (size_t k = 0; k < max_k; ++k, ++page) {
+ pages[k] = addr + (page * page_len);
+ status[k] = -EPERM;
+ }
- ret = move_pages(0, max_k, pages, nodes, status, numa_flags);
+ ret = move_pages(0, max_k, pages, nodes, status, numa_flags);
- if (ret < 0)
- return ret;
+ if (ret < 0)
+ return ret;
- if (ret > 0) {
- fprintf(stderr, "%lu pages were not migrated\n", ret);
- for (size_t k = 0; k < max_k; ++k) {
- if (status[k] < 0)
- fprintf(stderr,
- "Error while moving page %p to numa node %d: %u\n",
- pages[k], nodes[k], -status[k]);
- }
+ if (ret > 0) {
+ fprintf(stderr, "%lu pages were not migrated\n", ret);
+ for (size_t k = 0; k < max_k; ++k) {
+ if (status[k] < 0)
+ fprintf(stderr,
+ "Error while moving page %p to numa node %d: %u\n",
+ pages[k], nodes[k], -status[k]);
}
}
}
return 0;
}
-
-int rseq_mempool_init_numa(struct rseq_mempool *pool, int numa_flags)
-{
- struct rseq_mempool_range *range;
- int ret;
-
- if (!numa_flags)
- return 0;
- for (range = pool->ranges; range; range = range->next) {
- ret = rseq_mempool_range_init_numa(pool, range, numa_flags);
- if (ret)
- return ret;
- }
- return 0;
-}
#else
-int rseq_mempool_init_numa(struct rseq_mempool *pool __attribute__((unused)),
+int rseq_mempool_range_init_numa(void *addr __attribute__((unused)),
+ size_t len __attribute__((unused)),
+ int cpu __attribute__((unused)),
int numa_flags __attribute__((unused)))
{
- return 0;
+ errno = ENOSYS;
+ return -1;
}
#endif
{
size_t count;
- count = ((pool->percpu_stride >> pool->item_order) + BIT_PER_ULONG - 1) / BIT_PER_ULONG;
+ count = ((pool->attr.stride >> pool->item_order) + BIT_PER_ULONG - 1) / BIT_PER_ULONG;
/*
* Not being able to create the validation bitmap is an error
return;
for (range = pool->ranges; range; range = range->next) {
- total_item += pool->percpu_stride >> pool->item_order;
- total_never_allocated += (pool->percpu_stride - range->next_unused) >> pool->item_order;
+ total_item += pool->attr.stride >> pool->item_order;
+ total_never_allocated += (pool->attr.stride - range->next_unused) >> pool->item_order;
}
max_list_traversal = total_item - total_never_allocated;
if (!bitmap)
return;
- count = ((pool->percpu_stride >> pool->item_order) + BIT_PER_ULONG - 1) / BIT_PER_ULONG;
+ count = ((pool->attr.stride >> pool->item_order) + BIT_PER_ULONG - 1) / BIT_PER_ULONG;
/* Assert that all items in the pool were freed. */
for (size_t k = 0; k < count; ++k)
destroy_alloc_bitmap(pool, range);
/* range is a header located one page before the aligned mapping. */
return pool->attr.munmap_func(pool->attr.mmap_priv, range->header,
- (pool->percpu_stride * pool->max_nr_cpus) + rseq_get_page_len());
+ (pool->attr.stride * pool->attr.max_nr_cpus) + rseq_get_page_len());
}
/*
void *ptr;
if (len < page_size || alignment < page_size ||
- !is_pow2(len) || !is_pow2(alignment)) {
+ !is_pow2(alignment) || (len & (alignment - 1))) {
errno = EINVAL;
return NULL;
}
page_size = rseq_get_page_len();
base = aligned_mmap_anonymous(pool, page_size,
- pool->percpu_stride * pool->max_nr_cpus,
- pool->percpu_stride,
+ pool->attr.stride * pool->attr.max_nr_cpus,
+ pool->attr.stride,
&header, page_size);
if (!base)
return NULL;
if (create_alloc_bitmap(pool, range))
goto error_alloc;
}
+ if (pool->attr.init_set) {
+ int cpu;
+
+ for (cpu = 0; cpu < pool->attr.max_nr_cpus; cpu++) {
+ if (pool->attr.init_func(pool->attr.init_priv,
+ base + (pool->attr.stride * cpu),
+ pool->attr.stride, cpu)) {
+ goto error_alloc;
+ }
+ }
+ }
return range;
error_alloc:
}
struct rseq_mempool *rseq_mempool_create(const char *pool_name,
- size_t item_len, size_t percpu_stride, int max_nr_cpus,
- const struct rseq_mempool_attr *_attr)
+ size_t item_len, const struct rseq_mempool_attr *_attr)
{
struct rseq_mempool *pool;
struct rseq_mempool_attr attr = {};
}
item_len = 1UL << order;
- if (!percpu_stride)
- percpu_stride = RSEQ_PERCPU_STRIDE; /* Use default */
-
- if (max_nr_cpus < 0 || item_len > percpu_stride ||
- percpu_stride < (size_t) rseq_get_page_len() ||
- !is_pow2(percpu_stride)) {
- errno = EINVAL;
- return NULL;
- }
-
if (_attr)
memcpy(&attr, _attr, sizeof(attr));
if (!attr.mmap_set) {
attr.mmap_priv = NULL;
}
+ switch (attr.type) {
+ case MEMPOOL_TYPE_PERCPU:
+ if (attr.max_nr_cpus < 0) {
+ errno = EINVAL;
+ return NULL;
+ }
+ if (attr.max_nr_cpus == 0) {
+ /* Auto-detect */
+ attr.max_nr_cpus = rseq_get_max_nr_cpus();
+ if (attr.max_nr_cpus == 0) {
+ errno = EINVAL;
+ return NULL;
+ }
+ }
+ break;
+ case MEMPOOL_TYPE_GLOBAL:
+ /* Use a 1-cpu pool for global mempool type. */
+ attr.max_nr_cpus = 1;
+ break;
+ }
+ if (!attr.stride)
+ attr.stride = RSEQ_MEMPOOL_STRIDE; /* Use default */
+ if (item_len > attr.stride || attr.stride < (size_t) rseq_get_page_len() ||
+ !is_pow2(attr.stride)) {
+ errno = EINVAL;
+ return NULL;
+ }
+
pool = calloc(1, sizeof(struct rseq_mempool));
if (!pool)
return NULL;
memcpy(&pool->attr, &attr, sizeof(attr));
pthread_mutex_init(&pool->lock, NULL);
- pool->percpu_stride = percpu_stride;
- pool->max_nr_cpus = max_nr_cpus;
pool->item_len = item_len;
pool->item_order = order;
addr = (void __rseq_percpu *) (pool->ranges->base + item_offset);
goto end;
}
- if (pool->ranges->next_unused + pool->item_len > pool->percpu_stride) {
+ if (pool->ranges->next_unused + pool->item_len > pool->attr.stride) {
errno = ENOMEM;
addr = NULL;
goto end;
set_alloc_slot(pool, item_offset);
pthread_mutex_unlock(&pool->lock);
if (zeroed && addr)
- rseq_percpu_zero_item(pool, item_offset);
+ rseq_percpu_zero_item(pool, pool->ranges, item_offset);
return addr;
}
-void __rseq_percpu *rseq_percpu_malloc(struct rseq_mempool *pool)
+void __rseq_percpu *rseq_mempool_percpu_malloc(struct rseq_mempool *pool)
{
return __rseq_percpu_malloc(pool, false);
}
-void __rseq_percpu *rseq_percpu_zmalloc(struct rseq_mempool *pool)
+void __rseq_percpu *rseq_mempool_percpu_zmalloc(struct rseq_mempool *pool)
{
return __rseq_percpu_malloc(pool, true);
}
bitmap[k] &= ~mask;
}
-void librseq_percpu_free(void __rseq_percpu *_ptr, size_t percpu_stride)
+void librseq_mempool_percpu_free(void __rseq_percpu *_ptr, size_t stride)
{
uintptr_t ptr = (uintptr_t) _ptr;
- void *range_base = (void *) (ptr & (~(percpu_stride - 1)));
+ void *range_base = (void *) (ptr & (~(stride - 1)));
struct rseq_mempool_range *range = (struct rseq_mempool_range *) (range_base - RANGE_HEADER_OFFSET);
struct rseq_mempool *pool = range->pool;
- uintptr_t item_offset = ptr & (percpu_stride - 1);
+ uintptr_t item_offset = ptr & (stride - 1);
struct free_list_node *head, *item;
pthread_mutex_lock(&pool->lock);
return addr;
}
-void __rseq_percpu *rseq_percpu_mempool_set_malloc(struct rseq_mempool_set *pool_set, size_t len)
+void __rseq_percpu *rseq_mempool_set_percpu_malloc(struct rseq_mempool_set *pool_set, size_t len)
{
return __rseq_mempool_set_malloc(pool_set, len, false);
}
-void __rseq_percpu *rseq_percpu_mempool_set_zmalloc(struct rseq_mempool_set *pool_set, size_t len)
+void __rseq_percpu *rseq_mempool_set_percpu_zmalloc(struct rseq_mempool_set *pool_set, size_t len)
{
return __rseq_mempool_set_malloc(pool_set, len, true);
}
return 0;
}
+int rseq_mempool_attr_set_init(struct rseq_mempool_attr *attr,
+ int (*init_func)(void *priv, void *addr, size_t len, int cpu),
+ void *init_priv)
+{
+ if (!attr) {
+ errno = EINVAL;
+ return -1;
+ }
+ attr->init_set = true;
+ attr->init_func = init_func;
+ attr->init_priv = init_priv;
+ return 0;
+}
+
int rseq_mempool_attr_set_robust(struct rseq_mempool_attr *attr)
{
if (!attr) {
attr->robust_set = true;
return 0;
}
+
+int rseq_mempool_attr_set_percpu(struct rseq_mempool_attr *attr,
+ size_t stride, int max_nr_cpus)
+{
+ if (!attr) {
+ errno = EINVAL;
+ return -1;
+ }
+ attr->type = MEMPOOL_TYPE_PERCPU;
+ attr->stride = stride;
+ attr->max_nr_cpus = max_nr_cpus;
+ return 0;
+}
+
+int rseq_mempool_attr_set_global(struct rseq_mempool_attr *attr,
+ size_t stride)
+{
+ if (!attr) {
+ errno = EINVAL;
+ return -1;
+ }
+ attr->type = MEMPOOL_TYPE_GLOBAL;
+ attr->stride = stride;
+ attr->max_nr_cpus = 0;
+ return 0;
+}
+
+int rseq_mempool_get_max_nr_cpus(struct rseq_mempool *mempool)
+{
+ if (!mempool || mempool->attr.type != MEMPOOL_TYPE_PERCPU) {
+ errno = EINVAL;
+ return -1;
+ }
+ return mempool->attr.max_nr_cpus;
+}