// SPDX-License-Identifier: MIT
// SPDX-FileCopyrightText: 2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+// SPDX-FileCopyrightText: 2024 Olivier Dion <odion@efficios.com>
#include <rseq/mempool.h>
#include <sys/mman.h>
#define POOL_SET_NR_ENTRIES RSEQ_BITS_PER_LONG
+#define POOL_HEADER_NR_PAGES 2
+
/*
* Smallest allocation should hold enough space for a free list pointer.
*/
#define RANGE_HEADER_OFFSET sizeof(struct rseq_mempool_range)
#if RSEQ_BITS_PER_LONG == 64
-# define DEFAULT_POISON_VALUE 0x5555555555555555ULL
+# define DEFAULT_COW_INIT_POISON_VALUE 0x5555555555555555ULL
#else
-# define DEFAULT_POISON_VALUE 0x55555555UL
+# define DEFAULT_COW_INIT_POISON_VALUE 0x55555555UL
#endif
+/*
+ * Define the default COW_ZERO poison value as zero to prevent useless
+ * COW page allocation when writing poison values when freeing items.
+ */
+#define DEFAULT_COW_ZERO_POISON_VALUE 0x0
+
struct free_list_node;
struct free_list_node {
};
struct rseq_mempool_attr {
- bool mmap_set;
- void *(*mmap_func)(void *priv, size_t len);
- int (*munmap_func)(void *priv, void *ptr, size_t len);
- void *mmap_priv;
-
bool init_set;
int (*init_func)(void *priv, void *addr, size_t len, int cpu);
void *init_priv;
/*
* Memory layout of a mempool range:
- * - Header page (contains struct rseq_mempool_range at the very end),
- * - Base of the per-cpu data, starting with CPU 0,
+ * - Canary header page (for detection of destroy-after-fork of
+ * COW_INIT pool),
+ * - Header page (contains struct rseq_mempool_range at the
+ * very end),
+ * - Base of the per-cpu data, starting with CPU 0.
+ * Aliases with free-list for non-robust COW_ZERO pool.
* - CPU 1,
* ...
* - CPU max_nr_cpus - 1
- * - init values (unpopulated for RSEQ_MEMPOOL_POPULATE_ALL).
+ * - init values (only allocated for COW_INIT pool).
+ * Aliases with free-list for non-robust COW_INIT pool.
+ * - free list (for robust pool).
+ *
+ * The free list aliases the CPU 0 memory area for non-robust
+ * COW_ZERO pools. It aliases with init values for non-robust
+ * COW_INIT pools. It is located immediately after the init
+ * values for robust pools.
*/
void *header;
void *base;
/*
* The init values contains malloc_init/zmalloc values.
- * Pointer is NULL for RSEQ_MEMPOOL_POPULATE_ALL.
+ * Pointer is NULL for RSEQ_MEMPOOL_POPULATE_COW_ZERO.
*/
void *init;
size_t next_unused;
int item_order;
/*
- * The free list chains freed items on the CPU 0 address range.
- * We should rethink this decision if false sharing between
- * malloc/free from other CPUs and data accesses from CPU 0
- * becomes an issue. This is a NULL-terminated singly-linked
- * list.
+ * COW_INIT non-robust pools:
+ * The free list chains freed items on the init
+ * values address range.
+ *
+ * COW_ZERO non-robust pools:
+ * The free list chains freed items on the CPU 0
+ * address range. We should rethink this
+ * decision if false sharing between malloc/free
+ * from other CPUs and data accesses from CPU 0
+ * becomes an issue.
+ *
+ * Robust pools: The free list chains freed items in the
+ * address range dedicated for the free list.
+ *
+ * This is a NULL-terminated singly-linked list.
*/
struct free_list_node *free_list_head;
struct rseq_mempool *entries[POOL_SET_NR_ENTRIES];
};
-/*
- * This memfd is used to implement the user COW behavior for the page
- * protection scheme. memfd is a sparse virtual file. Its layout (in
- * offset from beginning of file) matches the process address space
- * (pointers directly converted to file offsets).
- */
-struct rseq_memfd {
- pthread_mutex_t lock;
- size_t reserved_size;
- unsigned int refcount;
- int fd;
-};
-
-static struct rseq_memfd memfd = {
- .lock = PTHREAD_MUTEX_INITIALIZER,
- .reserved_size = 0,
- .refcount = 0,
- .fd = -1,
-};
-
static
const char *get_pool_name(const struct rseq_mempool *pool)
{
{
void __rseq_percpu *p = (void __rseq_percpu *) node;
- if (pool->attr.populate_policy != RSEQ_MEMPOOL_POPULATE_ALL)
+ if (pool->attr.robust_set) {
+ /* Skip cpus. */
p -= pool->attr.max_nr_cpus * pool->attr.stride;
+ /* Skip init values */
+ if (pool->attr.populate_policy == RSEQ_MEMPOOL_POPULATE_COW_INIT)
+ p -= pool->attr.stride;
+
+ } else {
+ /* COW_INIT free list is in init values */
+ if (pool->attr.populate_policy == RSEQ_MEMPOOL_POPULATE_COW_INIT)
+ p -= pool->attr.max_nr_cpus * pool->attr.stride;
+ }
return p;
}
struct free_list_node *__rseq_percpu_to_free_list_ptr(const struct rseq_mempool *pool,
void __rseq_percpu *p)
{
- if (pool->attr.populate_policy != RSEQ_MEMPOOL_POPULATE_ALL)
+ if (pool->attr.robust_set) {
+ /* Skip cpus. */
p += pool->attr.max_nr_cpus * pool->attr.stride;
+ /* Skip init values */
+ if (pool->attr.populate_policy == RSEQ_MEMPOOL_POPULATE_COW_INIT)
+ p += pool->attr.stride;
+
+ } else {
+ /* COW_INIT free list is in init values */
+ if (pool->attr.populate_policy == RSEQ_MEMPOOL_POPULATE_COW_INIT)
+ p += pool->attr.max_nr_cpus * pool->attr.stride;
+ }
return (struct free_list_node *) p;
}
static
-off_t ptr_to_off_t(void *p)
+intptr_t rseq_cmp_item(void *p, size_t item_len, intptr_t cmp_value, intptr_t *unexpected_value)
{
- return (off_t) (uintptr_t) p;
-}
+ size_t offset;
+ intptr_t res = 0;
-static
-int memcmpbyte(const char *s, int c, size_t n)
-{
- int res = 0;
+ for (offset = 0; offset < item_len; offset += sizeof(uintptr_t)) {
+ intptr_t v = *((intptr_t *) (p + offset));
- while (n-- > 0)
- if ((res = *(s++) - c) != 0)
+ if ((res = v - cmp_value) != 0) {
+ if (unexpected_value)
+ *unexpected_value = v;
break;
+ }
+ }
return res;
}
init_p = __rseq_pool_range_init_ptr(range, item_offset);
if (init_p)
- memset(init_p, 0, pool->item_len);
+ bzero(init_p, pool->item_len);
for (i = 0; i < pool->attr.max_nr_cpus; i++) {
char *p = __rseq_pool_range_percpu_ptr(range, i,
item_offset, pool->attr.stride);
- /* Update propagated */
- if (init_p && !memcmpbyte(p, 0, pool->item_len))
+ /*
+ * If item is already zeroed, either because the
+ * init range update has propagated or because the
+ * content is already zeroed (e.g. zero page), don't
+ * write to the page. This eliminates useless COW over
+ * the zero page just for overwriting it with zeroes.
+ *
+ * This means zmalloc() in COW_ZERO policy pool do
+ * not trigger COW for CPUs which are not actively
+ * writing to the pool. This is however not the case for
+ * malloc_init() in populate-all pools if it populates
+ * non-zero content.
+ */
+ if (!rseq_cmp_item(p, pool->item_len, 0, NULL))
continue;
- memset(p, 0, pool->item_len);
+ bzero(p, pool->item_len);
}
}
char *p = __rseq_pool_range_percpu_ptr(range, i,
item_offset, pool->attr.stride);
- /* Update propagated */
- if (init_p && !memcmp(init_p, p, init_len))
+ /*
+ * If the update propagated through a shared mapping,
+ * or the item already has the correct content, skip
+ * writing it into the cpu item to eliminate useless
+ * COW of the page.
+ */
+ if (!memcmp(init_ptr, p, init_len))
continue;
memcpy(p, init_ptr, init_len);
}
char *p = __rseq_pool_range_percpu_ptr(range, i,
item_offset, pool->attr.stride);
- /* Update propagated */
- if (init_p && !memcmp(init_p, p, pool->item_len))
+ /*
+ * If the update propagated through a shared mapping,
+ * or the item already has the correct content, skip
+ * writing it into the cpu item to eliminate useless
+ * COW of the page.
+ *
+ * It is recommended to use zero as poison value for
+ * COW_ZERO pools to eliminate COW due to writing
+ * poison to CPU memory still backed by the zero page.
+ */
+ if (rseq_cmp_item(p, pool->item_len, poison, NULL) == 0)
continue;
rseq_poison_item(p, pool->item_len, poison);
}
/* Always inline for __builtin_return_address(0). */
static inline __attribute__((always_inline))
void rseq_check_poison_item(const struct rseq_mempool *pool, uintptr_t item_offset,
- void *p, size_t item_len, uintptr_t poison, bool skip_freelist_ptr)
+ void *p, size_t item_len, uintptr_t poison)
{
- size_t offset;
+ intptr_t unexpected_value;
- for (offset = 0; offset < item_len; offset += sizeof(uintptr_t)) {
- uintptr_t v;
+ if (rseq_cmp_item(p, item_len, poison, &unexpected_value) == 0)
+ return;
- /* Skip poison check for free-list pointer. */
- if (skip_freelist_ptr && offset == 0)
- continue;
- v = *((uintptr_t *) (p + offset));
- if (v != poison) {
- fprintf(stderr, "%s: Poison corruption detected (0x%lx) for pool: \"%s\" (%p), item offset: %zu, caller: %p.\n",
- __func__, (unsigned long) v, get_pool_name(pool), pool, item_offset, (void *) __builtin_return_address(0));
- abort();
- }
- }
+ fprintf(stderr, "%s: Poison corruption detected (0x%lx) for pool: \"%s\" (%p), item offset: %zu, caller: %p.\n",
+ __func__, (unsigned long) unexpected_value, get_pool_name(pool), pool, item_offset, (void *) __builtin_return_address(0));
+ abort();
}
/* Always inline for __builtin_return_address(0). */
return;
init_p = __rseq_pool_range_init_ptr(range, item_offset);
if (init_p)
- rseq_check_poison_item(pool, item_offset, init_p, pool->item_len, poison, true);
+ rseq_check_poison_item(pool, item_offset, init_p, pool->item_len, poison);
for (i = 0; i < pool->attr.max_nr_cpus; i++) {
char *p = __rseq_pool_range_percpu_ptr(range, i,
item_offset, pool->attr.stride);
- /*
- * When the free list is embedded in the init values
- * memory (populate none), it is visible from the init
- * values memory mapping as well as per-cpu private
- * mappings before they COW.
- *
- * When the free list is embedded in CPU 0 mapping
- * (populate all), only this CPU must skip the free list
- * nodes when checking poison.
- */
- rseq_check_poison_item(pool, item_offset, p, pool->item_len, poison,
- init_p == NULL ? (i == 0) : true);
+ rseq_check_poison_item(pool, item_offset, p, pool->item_len, poison);
}
}
}
#endif
-static
-void *default_mmap_func(void *priv __attribute__((unused)), size_t len)
-{
- void *base;
-
- base = mmap(NULL, len, PROT_READ | PROT_WRITE,
- MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
- if (base == MAP_FAILED)
- return NULL;
- return base;
-}
-
-static
-int default_munmap_func(void *priv __attribute__((unused)), void *ptr, size_t len)
-{
- return munmap(ptr, len);
-}
-
static
int create_alloc_bitmap(struct rseq_mempool *pool, struct rseq_mempool_range *range)
{
/* Always inline for __builtin_return_address(0). */
static inline __attribute__((always_inline))
-void check_free_list(const struct rseq_mempool *pool)
+void check_free_list(const struct rseq_mempool *pool, bool mapping_accessible)
{
size_t total_item = 0, total_never_allocated = 0, total_freed = 0,
max_list_traversal = 0, traversal_iteration = 0;
struct rseq_mempool_range *range;
- if (!pool->attr.robust_set)
+ if (!pool->attr.robust_set || !mapping_accessible)
return;
for (range = pool->range_list; range; range = range->next) {
/* Always inline for __builtin_return_address(0). */
static inline __attribute__((always_inline))
-void check_pool_poison(const struct rseq_mempool *pool)
+void check_pool_poison(const struct rseq_mempool *pool, bool mapping_accessible)
{
struct rseq_mempool_range *range;
- if (!pool->attr.robust_set)
+ if (!pool->attr.robust_set || !mapping_accessible)
return;
for (range = pool->range_list; range; range = range->next)
check_range_poison(pool, range);
/* Always inline for __builtin_return_address(0). */
static inline __attribute__((always_inline))
int rseq_mempool_range_destroy(struct rseq_mempool *pool,
- struct rseq_mempool_range *range)
+ struct rseq_mempool_range *range,
+ bool mapping_accessible)
{
- int ret = 0;
-
destroy_alloc_bitmap(pool, range);
-
- /*
- * Punch a hole into memfd where the init values used to be.
- */
- if (range->init) {
- ret = fallocate(memfd.fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
- ptr_to_off_t(range->init), pool->attr.stride);
- if (ret)
- return ret;
- range->init = NULL;
+ if (!mapping_accessible) {
+ /*
+ * Only the header pages are populated in the child
+ * process.
+ */
+ return munmap(range->header, POOL_HEADER_NR_PAGES * rseq_get_page_len());
}
-
- /* range is a header located one page before the aligned mapping. */
- return pool->attr.munmap_func(pool->attr.mmap_priv, range->mmap_addr, range->mmap_len);
+ return munmap(range->mmap_addr, range->mmap_len);
}
/*
* @pre_header before the mapping.
*/
static
-void *aligned_mmap_anonymous(struct rseq_mempool *pool,
- size_t page_size, size_t len, size_t alignment,
+void *aligned_mmap_anonymous(size_t page_size, size_t len, size_t alignment,
void **pre_header, size_t pre_header_len)
{
size_t minimum_page_count, page_count, extra, total_allocate = 0;
assert(page_count >= minimum_page_count);
- ptr = pool->attr.mmap_func(pool->attr.mmap_priv, page_count << page_order);
- if (!ptr)
+ ptr = mmap(NULL, page_count << page_order, PROT_READ | PROT_WRITE,
+ MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+ if (ptr == MAP_FAILED) {
+ ptr = NULL;
goto alloc_error;
+ }
total_allocate = page_count << page_order;
/* Unmap extra before. */
extra = offset_align((uintptr_t) ptr + pre_header_len, alignment);
assert(!(extra & (page_size - 1)));
- if (pool->attr.munmap_func(pool->attr.mmap_priv, ptr, extra)) {
+ if (munmap(ptr, extra)) {
perror("munmap");
abort();
}
/* Unmap extra after. */
extra_ptr = ptr + (minimum_page_count << page_order);
extra = (page_count - minimum_page_count) << page_order;
- if (pool->attr.munmap_func(pool->attr.mmap_priv, extra_ptr, extra)) {
+ if (munmap(extra_ptr, extra)) {
perror("munmap");
abort();
}
}
static
-int rseq_memfd_reserve_init(void *init, size_t init_len)
+int rseq_memfd_create_init(const char *poolname, size_t init_len)
{
- int ret = 0;
- size_t reserve_len;
-
- pthread_mutex_lock(&memfd.lock);
- reserve_len = (size_t) ptr_to_off_t(init) + init_len;
- if (reserve_len > memfd.reserved_size) {
- if (ftruncate(memfd.fd, (off_t) reserve_len)) {
- ret = -1;
- goto unlock;
- }
- memfd.reserved_size = reserve_len;
+ int fd;
+ char buf[249]; /* Limit is 249 bytes. */
+ const char *name;
+
+ if (poolname) {
+ snprintf(buf, sizeof(buf), "%s:rseq-mempool", poolname);
+ name = buf;
+ } else {
+ name = "<anonymous>:rseq-mempool";
}
-unlock:
- pthread_mutex_unlock(&memfd.lock);
- return ret;
+
+ fd = memfd_create(name, MFD_CLOEXEC);
+ if (fd < 0) {
+ perror("memfd_create");
+ goto end;
+ }
+ if (ftruncate(fd, (off_t) init_len)) {
+ if (close(fd))
+ perror("close");
+ fd = -1;
+ goto end;
+ }
+end:
+ return fd;
+}
+
+static
+void rseq_memfd_close(int fd)
+{
+ if (fd < 0)
+ return;
+ if (close(fd))
+ perror("close");
}
static
void *header;
void *base;
size_t range_len; /* Range len excludes header. */
+ size_t header_len;
+ int memfd = -1;
if (pool->attr.max_nr_ranges &&
pool->nr_ranges >= pool->attr.max_nr_ranges) {
}
page_size = rseq_get_page_len();
+ header_len = POOL_HEADER_NR_PAGES * page_size;
range_len = pool->attr.stride * pool->attr.max_nr_cpus;
- if (pool->attr.populate_policy != RSEQ_MEMPOOL_POPULATE_ALL)
+ if (pool->attr.populate_policy == RSEQ_MEMPOOL_POPULATE_COW_INIT)
range_len += pool->attr.stride; /* init values */
- base = aligned_mmap_anonymous(pool, page_size,
- range_len,
- pool->attr.stride,
- &header, page_size);
+ if (pool->attr.robust_set)
+ range_len += pool->attr.stride; /* dedicated free list */
+ base = aligned_mmap_anonymous(page_size, range_len,
+ pool->attr.stride, &header, header_len);
if (!base)
return NULL;
range = (struct rseq_mempool_range *) (base - RANGE_HEADER_OFFSET);
range->header = header;
range->base = base;
range->mmap_addr = header;
- range->mmap_len = page_size + range_len;
+ range->mmap_len = header_len + range_len;
- if (pool->attr.populate_policy != RSEQ_MEMPOOL_POPULATE_ALL) {
+ if (pool->attr.populate_policy == RSEQ_MEMPOOL_POPULATE_COW_INIT) {
range->init = base + (pool->attr.stride * pool->attr.max_nr_cpus);
/* Populate init values pages from memfd */
- if (rseq_memfd_reserve_init(range->init, pool->attr.stride))
+ memfd = rseq_memfd_create_init(pool->name, pool->attr.stride);
+ if (memfd < 0)
goto error_alloc;
if (mmap(range->init, pool->attr.stride, PROT_READ | PROT_WRITE,
- MAP_SHARED | MAP_FIXED, memfd.fd,
- ptr_to_off_t(range->init)) != (void *) range->init) {
+ MAP_SHARED | MAP_FIXED, memfd, 0) != (void *) range->init)
goto error_alloc;
- }
assert(pool->attr.type == MEMPOOL_TYPE_PERCPU);
/*
* Map per-cpu memory as private COW mappings of init values.
size_t len = pool->attr.stride;
if (mmap(p, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_FIXED,
- memfd.fd, ptr_to_off_t(range->init)) != (void *) p) {
+ memfd, 0) != (void *) p)
goto error_alloc;
- }
}
}
+ /*
+ * The init values shared mapping should not be shared
+ * with the children processes across fork. Prevent the
+ * whole mapping from being used across fork.
+ */
+ if (madvise(base, range_len, MADV_DONTFORK))
+ goto error_alloc;
+
+ /*
+ * Write 0x1 in first byte of header first page, which
+ * will be WIPEONFORK (and thus cleared) in children
+ * processes. Used to find out if pool destroy is called
+ * from a child process after fork.
+ */
+ *((char *) header) = 0x1;
+ if (madvise(header, page_size, MADV_WIPEONFORK))
+ goto error_alloc;
+
+ /*
+ * The second header page contains the struct
+ * rseq_mempool_range, which is needed by pool destroy.
+ * Leave this anonymous page populated (COW) in child
+ * processes.
+ */
+ rseq_memfd_close(memfd);
+ memfd = -1;
}
if (pool->attr.robust_set) {
return range;
error_alloc:
- (void) rseq_mempool_range_destroy(pool, range);
+ rseq_memfd_close(memfd);
+ (void) rseq_mempool_range_destroy(pool, range, true);
return NULL;
}
static
-int rseq_mempool_memfd_ref(struct rseq_mempool *pool)
+bool pool_mappings_accessible(struct rseq_mempool *pool)
{
- int ret = 0;
-
- if (pool->attr.populate_policy == RSEQ_MEMPOOL_POPULATE_ALL)
- return 0;
-
- pthread_mutex_lock(&memfd.lock);
- if (memfd.refcount == 0) {
- memfd.fd = memfd_create("mempool", MFD_CLOEXEC);
- if (memfd.fd < 0) {
- perror("memfd_create");
- ret = -1;
- goto unlock;
- }
- }
- memfd.refcount++;
-unlock:
- pthread_mutex_unlock(&memfd.lock);
- return ret;
-}
-
-static
-void rseq_mempool_memfd_unref(struct rseq_mempool *pool)
-{
- if (pool->attr.populate_policy == RSEQ_MEMPOOL_POPULATE_ALL)
- return;
+ struct rseq_mempool_range *range;
+ size_t page_size;
+ char *addr;
- pthread_mutex_lock(&memfd.lock);
- if (memfd.refcount == 1) {
- if (close(memfd.fd)) {
- perror("close");
- abort();
- }
- memfd.fd = -1;
- memfd.reserved_size = 0;
- }
- memfd.refcount--;
- pthread_mutex_unlock(&memfd.lock);
+ if (pool->attr.populate_policy != RSEQ_MEMPOOL_POPULATE_COW_INIT)
+ return true;
+ range = pool->range_list;
+ if (!range)
+ return true;
+ page_size = rseq_get_page_len();
+ /*
+ * Header first page is one page before the page containing the
+ * range structure.
+ */
+ addr = (char *) ((uintptr_t) range & ~(page_size - 1)) - page_size;
+ /*
+ * Look for 0x1 first byte marker in header first page.
+ */
+ if (*addr != 0x1)
+ return false;
+ return true;
}
int rseq_mempool_destroy(struct rseq_mempool *pool)
{
struct rseq_mempool_range *range, *next_range;
+ bool mapping_accessible;
int ret = 0;
if (!pool)
return 0;
- check_free_list(pool);
- check_pool_poison(pool);
+
+ /*
+ * Validate that the pool mappings are accessible before doing
+ * free list/poison validation and unmapping ranges. This allows
+ * calling pool destroy in child process after a fork for COW_INIT
+ * pools to free pool resources.
+ */
+ mapping_accessible = pool_mappings_accessible(pool);
+
+ check_free_list(pool, mapping_accessible);
+ check_pool_poison(pool, mapping_accessible);
+
/* Iteration safe against removal. */
for (range = pool->range_list; range && (next_range = range->next, 1); range = next_range) {
- if (rseq_mempool_range_destroy(pool, range))
+ if (rseq_mempool_range_destroy(pool, range, mapping_accessible))
goto end;
/* Update list head to keep list coherent in case of partial failure. */
pool->range_list = next_range;
}
- rseq_mempool_memfd_unref(pool);
pthread_mutex_destroy(&pool->lock);
free(pool->name);
free(pool);
if (_attr)
memcpy(&attr, _attr, sizeof(attr));
- if (!attr.mmap_set) {
- attr.mmap_func = default_mmap_func;
- attr.munmap_func = default_munmap_func;
- attr.mmap_priv = NULL;
+
+ /*
+ * Validate that the pool populate policy requested is known.
+ */
+ switch (attr.populate_policy) {
+ case RSEQ_MEMPOOL_POPULATE_COW_INIT:
+ break;
+ case RSEQ_MEMPOOL_POPULATE_COW_ZERO:
+ break;
+ default:
+ errno = EINVAL;
+ return NULL;
}
switch (attr.type) {
break;
case MEMPOOL_TYPE_GLOBAL:
/* Override populate policy for global type. */
- attr.populate_policy = RSEQ_MEMPOOL_POPULATE_ALL;
+ if (attr.populate_policy == RSEQ_MEMPOOL_POPULATE_COW_INIT)
+ attr.populate_policy = RSEQ_MEMPOOL_POPULATE_COW_ZERO;
/* Use a 1-cpu pool for global mempool type. */
attr.max_nr_cpus = 1;
break;
attr.stride = RSEQ_MEMPOOL_STRIDE; /* Use default */
if (attr.robust_set && !attr.poison_set) {
attr.poison_set = true;
- attr.poison = DEFAULT_POISON_VALUE;
+ if (attr.populate_policy == RSEQ_MEMPOOL_POPULATE_COW_INIT)
+ attr.poison = DEFAULT_COW_INIT_POISON_VALUE;
+ else
+ attr.poison = DEFAULT_COW_ZERO_POISON_VALUE;
}
if (item_len > attr.stride || attr.stride < (size_t) rseq_get_page_len() ||
!is_pow2(attr.stride)) {
pool->item_len = item_len;
pool->item_order = order;
- if (rseq_mempool_memfd_ref(pool))
- goto error_alloc;
-
pool->range_list = rseq_mempool_range_create(pool);
if (!pool->range_list)
goto error_alloc;
item = __rseq_percpu_to_free_list_ptr(pool, _ptr);
/*
* Setting the next pointer will overwrite the first uintptr_t
- * poison for either CPU 0 (populate all) or init data (populate
- * none).
+ * poison for either CPU 0 (COW_ZERO, non-robust), or init data
+ * (COW_INIT, non-robust).
*/
item->next = head;
pool->free_list_head = item;
free(attr);
}
-int rseq_mempool_attr_set_mmap(struct rseq_mempool_attr *attr,
- void *(*mmap_func)(void *priv, size_t len),
- int (*munmap_func)(void *priv, void *ptr, size_t len),
- void *mmap_priv)
-{
- if (!attr) {
- errno = EINVAL;
- return -1;
- }
- attr->mmap_set = true;
- attr->mmap_func = mmap_func;
- attr->munmap_func = munmap_func;
- attr->mmap_priv = mmap_priv;
- return 0;
-}
-
int rseq_mempool_attr_set_init(struct rseq_mempool_attr *attr,
int (*init_func)(void *priv, void *addr, size_t len, int cpu),
void *init_priv)
attr->init_set = true;
attr->init_func = init_func;
attr->init_priv = init_priv;
+ attr->populate_policy = RSEQ_MEMPOOL_POPULATE_COW_INIT;
return 0;
}
projects / librseq.git / blobdiff