projects
/
librseq.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
mempool: init_func can return an error
[librseq.git]
/
src
/
rseq-mempool.c
diff --git
a/src/rseq-mempool.c
b/src/rseq-mempool.c
index eb2d0b1c75a86b83dc0556bf9693db113a46f84e..5823ae59be9e2cccff60c77ef00566c95b9c7b36 100644
(file)
--- a/
src/rseq-mempool.c
+++ b/
src/rseq-mempool.c
@@
-20,7
+20,7
@@
#endif
#include "rseq-utils.h"
#endif
#include "rseq-utils.h"
-#include
"smp.h"
+#include
<rseq/rseq.h>
/*
* rseq-mempool.c: rseq CPU-Local Storage (CLS) memory allocator.
/*
* rseq-mempool.c: rseq CPU-Local Storage (CLS) memory allocator.
@@
-75,6
+75,10
@@
struct rseq_mempool_attr {
int (*munmap_func)(void *priv, void *ptr, size_t len);
void *mmap_priv;
int (*munmap_func)(void *priv, void *ptr, size_t len);
void *mmap_priv;
+ bool init_set;
+ int (*init_func)(void *priv, void *addr, size_t len, int cpu);
+ void *init_priv;
+
bool robust_set;
enum mempool_type type;
bool robust_set;
enum mempool_type type;
@@
-132,7
+136,6
@@
static
void *__rseq_pool_range_percpu_ptr(struct rseq_mempool_range *range, int cpu,
uintptr_t item_offset, size_t stride)
{
void *__rseq_pool_range_percpu_ptr(struct rseq_mempool_range *range, int cpu,
uintptr_t item_offset, size_t stride)
{
- /* TODO: Implement multi-ranges support. */
return range->base + (stride * cpu) + item_offset;
}
return range->base + (stride * cpu) + item_offset;
}
@@
-150,83
+153,68
@@
void rseq_percpu_zero_item(struct rseq_mempool *pool,
}
#ifdef HAVE_LIBNUMA
}
#ifdef HAVE_LIBNUMA
-static
-int rseq_mempool_range_init_numa(struct rseq_mempool *pool, struct rseq_mempool_range *range, int numa_flags)
+int rseq_mempool_range_init_numa(void *addr, size_t len, int cpu, int numa_flags)
{
unsigned long nr_pages, page_len;
{
unsigned long nr_pages, page_len;
+ int status[MOVE_PAGES_BATCH_SIZE];
+ int nodes[MOVE_PAGES_BATCH_SIZE];
+ void *pages[MOVE_PAGES_BATCH_SIZE];
long ret;
long ret;
- int cpu;
- if (!numa_flags)
- return 0;
+ if (!numa_flags) {
+ errno = EINVAL;
+ return -1;
+ }
page_len = rseq_get_page_len();
page_len = rseq_get_page_len();
- nr_pages = pool->attr.stride >> rseq_get_count_order_ulong(page_len);
- for (cpu = 0; cpu < pool->attr.max_nr_cpus; cpu++) {
+ nr_pages = len >> rseq_get_count_order_ulong(page_len);
-
int status[MOVE_PAGES_BATCH_SIZE]
;
- int nodes[MOVE_PAGES_BATCH_SIZE];
-
void *pages[MOVE_PAGES_BATCH_SIZE]
;
+
nodes[0] = numa_node_of_cpu(cpu)
;
+ if (nodes[0] < 0)
+
return -1
;
- nodes[0] = numa_node_of_cpu(cpu);
- if (nodes[0] < 0)
- continue;
- for (size_t k = 1; k < RSEQ_ARRAY_SIZE(nodes); ++k) {
- nodes[k] = nodes[0];
- }
+ for (size_t k = 1; k < RSEQ_ARRAY_SIZE(nodes); ++k) {
+ nodes[k] = nodes[0];
+ }
-
for (unsigned long page = 0; page < nr_pages;) {
+ for (unsigned long page = 0; page < nr_pages;) {
-
size_t max_k = RSEQ_ARRAY_SIZE(pages);
-
size_t left = nr_pages - page;
+ size_t max_k = RSEQ_ARRAY_SIZE(pages);
+ size_t left = nr_pages - page;
-
if (left < max_k) {
-
max_k = left;
-
}
+ if (left < max_k) {
+ max_k = left;
+ }
- for (size_t k = 0; k < max_k; ++k, ++page) {
- pages[k] = __rseq_pool_range_percpu_ptr(range, cpu,
- page * page_len, pool->attr.stride);
- status[k] = -EPERM;
- }
+ for (size_t k = 0; k < max_k; ++k, ++page) {
+ pages[k] = addr + (page * page_len);
+ status[k] = -EPERM;
+ }
-
ret = move_pages(0, max_k, pages, nodes, status, numa_flags);
+ ret = move_pages(0, max_k, pages, nodes, status, numa_flags);
-
if (ret < 0)
-
return ret;
+ if (ret < 0)
+ return ret;
- if (ret > 0) {
- fprintf(stderr, "%lu pages were not migrated\n", ret);
- for (size_t k = 0; k < max_k; ++k) {
- if (status[k] < 0)
- fprintf(stderr,
- "Error while moving page %p to numa node %d: %u\n",
- pages[k], nodes[k], -status[k]);
- }
+ if (ret > 0) {
+ fprintf(stderr, "%lu pages were not migrated\n", ret);
+ for (size_t k = 0; k < max_k; ++k) {
+ if (status[k] < 0)
+ fprintf(stderr,
+ "Error while moving page %p to numa node %d: %u\n",
+ pages[k], nodes[k], -status[k]);
}
}
}
return 0;
}
}
}
}
return 0;
}
-
-int rseq_mempool_init_numa(struct rseq_mempool *pool, int numa_flags)
-{
- struct rseq_mempool_range *range;
- int ret;
-
- if (!numa_flags)
- return 0;
- for (range = pool->ranges; range; range = range->next) {
- ret = rseq_mempool_range_init_numa(pool, range, numa_flags);
- if (ret)
- return ret;
- }
- return 0;
-}
#else
#else
-int rseq_mempool_init_numa(struct rseq_mempool *pool __attribute__((unused)),
+int rseq_mempool_range_init_numa(void *addr __attribute__((unused)),
+ size_t len __attribute__((unused)),
+ int cpu __attribute__((unused)),
int numa_flags __attribute__((unused)))
{
int numa_flags __attribute__((unused)))
{
- return 0;
+ errno = ENOSYS;
+ return -1;
}
#endif
}
#endif
@@
-384,7
+372,7
@@
void *aligned_mmap_anonymous(struct rseq_mempool *pool,
void *ptr;
if (len < page_size || alignment < page_size ||
void *ptr;
if (len < page_size || alignment < page_size ||
- !is_pow2(
len) || !is_pow2(alignment
)) {
+ !is_pow2(
alignment) || (len & (alignment - 1)
)) {
errno = EINVAL;
return NULL;
}
errno = EINVAL;
return NULL;
}
@@
-476,6
+464,17
@@
struct rseq_mempool_range *rseq_mempool_range_create(struct rseq_mempool *pool)
if (create_alloc_bitmap(pool, range))
goto error_alloc;
}
if (create_alloc_bitmap(pool, range))
goto error_alloc;
}
+ if (pool->attr.init_set) {
+ int cpu;
+
+ for (cpu = 0; cpu < pool->attr.max_nr_cpus; cpu++) {
+ if (pool->attr.init_func(pool->attr.init_priv,
+ base + (pool->attr.stride * cpu),
+ pool->attr.stride, cpu)) {
+ goto error_alloc;
+ }
+ }
+ }
return range;
error_alloc:
return range;
error_alloc:
@@
-540,7
+539,7
@@
struct rseq_mempool *rseq_mempool_create(const char *pool_name,
}
if (attr.max_nr_cpus == 0) {
/* Auto-detect */
}
if (attr.max_nr_cpus == 0) {
/* Auto-detect */
- attr.max_nr_cpus =
get_possible_cpus_array_len
();
+ attr.max_nr_cpus =
rseq_get_max_nr_cpus
();
if (attr.max_nr_cpus == 0) {
errno = EINVAL;
return NULL;
if (attr.max_nr_cpus == 0) {
errno = EINVAL;
return NULL;
@@
-826,6
+825,20
@@
int rseq_mempool_attr_set_mmap(struct rseq_mempool_attr *attr,
return 0;
}
return 0;
}
+int rseq_mempool_attr_set_init(struct rseq_mempool_attr *attr,
+ int (*init_func)(void *priv, void *addr, size_t len, int cpu),
+ void *init_priv)
+{
+ if (!attr) {
+ errno = EINVAL;
+ return -1;
+ }
+ attr->init_set = true;
+ attr->init_func = init_func;
+ attr->init_priv = init_priv;
+ return 0;
+}
+
int rseq_mempool_attr_set_robust(struct rseq_mempool_attr *attr)
{
if (!attr) {
int rseq_mempool_attr_set_robust(struct rseq_mempool_attr *attr)
{
if (!attr) {
@@
-861,3
+874,12
@@
int rseq_mempool_attr_set_global(struct rseq_mempool_attr *attr,
attr->max_nr_cpus = 0;
return 0;
}
attr->max_nr_cpus = 0;
return 0;
}
+
+int rseq_mempool_get_max_nr_cpus(struct rseq_mempool *mempool)
+{
+ if (!mempool || mempool->attr.type != MEMPOOL_TYPE_PERCPU) {
+ errno = EINVAL;
+ return -1;
+ }
+ return mempool->attr.max_nr_cpus;
+}
This page took
0.025143 seconds
and
4
git commands to generate.