+ total_allocate = page_count << page_order;
+
+ if (!(((uintptr_t) ptr + pre_header_len) & (alignment - 1))) {
+ /* Pointer is already aligned. ptr points to pre_header. */
+ goto out;
+ }
+
+ /* Unmap extra before. */
+ extra = offset_align((uintptr_t) ptr + pre_header_len, alignment);
+ assert(!(extra & (page_size - 1)));
+ if (munmap(ptr, extra)) {
+ perror("munmap");
+ abort();
+ }
+ total_allocate -= extra;
+ ptr += extra; /* ptr points to pre_header */
+ page_count -= extra >> page_order;
+out:
+ assert(page_count >= minimum_page_count);
+
+ if (page_count > minimum_page_count) {
+ void *extra_ptr;
+
+ /* Unmap extra after. */
+ extra_ptr = ptr + (minimum_page_count << page_order);
+ extra = (page_count - minimum_page_count) << page_order;
+ if (munmap(extra_ptr, extra)) {
+ perror("munmap");
+ abort();
+ }
+ total_allocate -= extra;
+ }
+
+ assert(!(((uintptr_t)ptr + pre_header_len) & (alignment - 1)));
+ assert(total_allocate == len + pre_header_len);
+
+alloc_error:
+ if (ptr) {
+ if (pre_header)
+ *pre_header = ptr;
+ ptr += pre_header_len;
+ }
+ return ptr;
+}
+
+static
+int rseq_memfd_create_init(const char *poolname, size_t init_len)
+{
+ int fd;
+ char buf[249]; /* Limit is 249 bytes. */
+ const char *name;
+
+ if (poolname) {
+ snprintf(buf, sizeof(buf), "%s:rseq-mempool", poolname);
+ name = buf;
+ } else {
+ name = "<anonymous>:rseq-mempool";
+ }
+
+ fd = memfd_create(name, MFD_CLOEXEC);
+ if (fd < 0) {
+ perror("memfd_create");
+ goto end;
+ }
+ if (ftruncate(fd, (off_t) init_len)) {
+ if (close(fd))
+ perror("close");
+ fd = -1;
+ goto end;
+ }
+end:
+ return fd;
+}
+
+static
+void rseq_memfd_close(int fd)
+{
+ if (fd < 0)
+ return;
+ if (close(fd))
+ perror("close");
+}
+
+static
+struct rseq_mempool_range *rseq_mempool_range_create(struct rseq_mempool *pool)
+{
+ struct rseq_mempool_range *range;
+ unsigned long page_size;
+ void *header;
+ void *base;
+ size_t range_len; /* Range len excludes header. */
+ size_t header_len;
+ int memfd = -1;
+
+ if (pool->attr.max_nr_ranges &&
+ pool->nr_ranges >= pool->attr.max_nr_ranges) {
+ errno = ENOMEM;
+ return NULL;
+ }
+ page_size = rseq_get_page_len();
+
+ header_len = POOL_HEADER_NR_PAGES * page_size;
+ range_len = pool->attr.stride * pool->attr.max_nr_cpus;
+ if (pool->attr.populate_policy == RSEQ_MEMPOOL_POPULATE_COW_INIT)
+ range_len += pool->attr.stride; /* init values */
+ if (pool->attr.robust_set)
+ range_len += pool->attr.stride; /* dedicated free list */
+ base = aligned_mmap_anonymous(page_size, range_len,
+ pool->attr.stride, &header, header_len);