-/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
+/* SPDX-License-Identifier: MIT */
+/* SPDX-FileCopyrightText: 2016-2022 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
+
/*
- * rseq.h
- *
- * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
+ * rseq/rseq.h
*/
-#ifndef RSEQ_H
-#define RSEQ_H
+#ifndef _RSEQ_RSEQ_H
+#define _RSEQ_RSEQ_H
#include <stdint.h>
#include <stdbool.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
-#include <sched.h>
-#include <linux/rseq.h>
+#include <stddef.h>
+#include <assert.h>
+
+#include <rseq/abi.h>
+#include <rseq/compiler.h>
+#include <rseq/inject.h>
+#include <rseq/thread-pointer.h>
+#include <rseq/utils.h>
+
+enum rseq_mo {
+ RSEQ_MO_RELAXED = 0,
+ RSEQ_MO_CONSUME = 1, /* Unused */
+ RSEQ_MO_ACQUIRE = 2, /* Unused */
+ RSEQ_MO_RELEASE = 3,
+ RSEQ_MO_ACQ_REL = 4, /* Unused */
+ RSEQ_MO_SEQ_CST = 5, /* Unused */
+};
+
+enum rseq_percpu_mode {
+ RSEQ_PERCPU_CPU_ID = 0,
+ RSEQ_PERCPU_MM_CID = 1,
+};
+
+enum rseq_available_query {
+ RSEQ_AVAILABLE_QUERY_KERNEL = 0,
+ RSEQ_AVAILABLE_QUERY_LIBC = 1,
+};
/*
- * Empty code injection macros, override when testing.
- * It is important to consider that the ASM injection macros need to be
- * fully reentrant (e.g. do not modify the stack).
+ * User code can define RSEQ_GET_ABI_OVERRIDE to override the
+ * rseq_get_abi() implementation, for instance to use glibc's symbols
+ * directly.
*/
-#ifndef RSEQ_INJECT_ASM
-#define RSEQ_INJECT_ASM(n)
-#endif
+#ifndef RSEQ_GET_ABI_OVERRIDE
-#ifndef RSEQ_INJECT_C
-#define RSEQ_INJECT_C(n)
-#endif
+# ifdef __cplusplus
+extern "C" {
+# endif
-#ifndef RSEQ_INJECT_INPUT
-#define RSEQ_INJECT_INPUT
-#endif
+/* Offset from the thread pointer to the rseq area. */
+extern ptrdiff_t rseq_offset;
-#ifndef RSEQ_INJECT_CLOBBER
-#define RSEQ_INJECT_CLOBBER
-#endif
+/*
+ * Size of the registered rseq area. 0 if the registration was
+ * unsuccessful.
+ */
+extern unsigned int rseq_size;
-#ifndef RSEQ_INJECT_FAILED
-#define RSEQ_INJECT_FAILED
-#endif
+/* Flags used during rseq registration. */
+extern unsigned int rseq_flags;
-#ifdef __cplusplus
-extern "C" {
-#endif
+/*
+ * rseq feature size supported by the kernel. 0 if the registration was
+ * unsuccessful.
+ */
+extern unsigned int rseq_feature_size;
-extern __thread struct rseq __rseq_abi;
-extern int __rseq_handled;
+/*
+ * Returns a pointer to the rseq area.
+ */
+static inline __attribute__((always_inline))
+struct rseq_abi *rseq_get_abi(void)
+{
+ return (struct rseq_abi *) ((uintptr_t) rseq_thread_pointer() + rseq_offset);
+}
-#ifdef __cplusplus
+# ifdef __cplusplus
}
-#endif
+# endif
+
+#endif /* RSEQ_GET_ABI_OVERRIDE */
+
+
+/*
+ * Architecture specific.
+ */
+#include <rseq/arch.h>
-#define rseq_likely(x) __builtin_expect(!!(x), 1)
-#define rseq_unlikely(x) __builtin_expect(!!(x), 0)
-#define rseq_barrier() __asm__ __volatile__("" : : : "memory")
-
-#define RSEQ_ACCESS_ONCE(x) (*(__volatile__ __typeof__(x) *)&(x))
-#define RSEQ_WRITE_ONCE(x, v) __extension__ ({ RSEQ_ACCESS_ONCE(x) = (v); })
-#define RSEQ_READ_ONCE(x) RSEQ_ACCESS_ONCE(x)
-
-#define __rseq_str_1(x) #x
-#define __rseq_str(x) __rseq_str_1(x)
-
-#define rseq_log(fmt, args...) \
- fprintf(stderr, fmt "(in %s() at " __FILE__ ":" __rseq_str(__LINE__)"\n", \
- ## args, __func__)
-
-#define rseq_bug(fmt, args...) \
- do { \
- rseq_log(fmt, ##args); \
- abort(); \
- } while (0)
-
-#if defined(__x86_64__) || defined(__i386__)
-#include <rseq/rseq-x86.h>
-#elif defined(__ARMEL__) || defined(__ARMEB__)
-#include <rseq/rseq-arm.h>
-#elif defined (__AARCH64EL__)
-#include <rseq/rseq-arm64.h>
-#elif defined(__PPC__)
-#include <rseq/rseq-ppc.h>
-#elif defined(__mips__)
-#include <rseq/rseq-mips.h>
-#elif defined(__s390__)
-#include <rseq/rseq-s390.h>
-#else
-#error unsupported target
-#endif
#ifdef __cplusplus
extern "C" {
*/
int32_t rseq_fallback_current_cpu(void);
-int rseq_available(void);
+/*
+ * Restartable sequence fallback for reading the current node number.
+ */
+int32_t rseq_fallback_current_node(void);
+
+/*
+ * Returns true if rseq is supported.
+ */
+bool rseq_available(unsigned int query);
/*
* Values returned can be either the current CPU number, -1 (rseq is
* uninitialized), or -2 (rseq initialization has failed).
*/
-static inline int32_t rseq_current_cpu_raw(void)
+static inline __attribute__((always_inline))
+int32_t rseq_current_cpu_raw(void)
{
- return RSEQ_READ_ONCE(__rseq_abi.cpu_id);
+ return RSEQ_READ_ONCE(rseq_get_abi()->cpu_id);
}
/*
* return value of rseq_current_cpu_raw() if the rseq asm sequence
* does not need to be invoked.
*/
-static inline uint32_t rseq_cpu_start(void)
+static inline __attribute__((always_inline))
+uint32_t rseq_cpu_start(void)
{
- return RSEQ_READ_ONCE(__rseq_abi.cpu_id_start);
+ return RSEQ_READ_ONCE(rseq_get_abi()->cpu_id_start);
}
-static inline uint32_t rseq_current_cpu(void)
+static inline __attribute__((always_inline))
+uint32_t rseq_current_cpu(void)
{
int32_t cpu;
return cpu;
}
-static inline void rseq_clear_rseq_cs(void)
+static inline __attribute__((always_inline))
+bool rseq_node_id_available(void)
{
-#ifdef __LP64__
- RSEQ_WRITE_ONCE(__rseq_abi.rseq_cs.ptr, 0);
-#else
- RSEQ_WRITE_ONCE(__rseq_abi.rseq_cs.ptr.ptr32, 0);
-#endif
+ return (int) rseq_feature_size >= (int) rseq_offsetofend(struct rseq_abi, node_id);
+}
+
+/*
+ * Current NUMA node number.
+ */
+static inline __attribute__((always_inline))
+uint32_t rseq_current_node_id(void)
+{
+ assert(rseq_node_id_available());
+ return RSEQ_READ_ONCE(rseq_get_abi()->node_id);
+}
+
+static inline __attribute__((always_inline))
+bool rseq_mm_cid_available(void)
+{
+ return (int) rseq_feature_size >= (int) rseq_offsetofend(struct rseq_abi, mm_cid);
+}
+
+static inline __attribute__((always_inline))
+uint32_t rseq_current_mm_cid(void)
+{
+ return RSEQ_READ_ONCE(rseq_get_abi()->mm_cid);
+}
+
+static inline __attribute__((always_inline))
+void rseq_clear_rseq_cs(void)
+{
+ RSEQ_WRITE_ONCE(rseq_get_abi()->rseq_cs.arch.ptr, 0);
}
/*
* rseq_prepare_unload() should be invoked by each thread executing a rseq
* critical section at least once between their last critical section and
- * library unload of the library defining the rseq critical section
- * (struct rseq_cs). This also applies to use of rseq in code generated by
- * JIT: rseq_prepare_unload() should be invoked at least once by each
- * thread executing a rseq critical section before reclaim of the memory
- * holding the struct rseq_cs.
+ * library unload of the library defining the rseq critical section (struct
+ * rseq_cs) or the code referred to by the struct rseq_cs start_ip and
+ * post_commit_offset fields. This also applies to use of rseq in code
+ * generated by JIT: rseq_prepare_unload() should be invoked at least once by
+ * each thread executing a rseq critical section before reclaim of the memory
+ * holding the struct rseq_cs or reclaim of the code pointed to by struct
+ * rseq_cs start_ip and post_commit_offset fields.
*/
-static inline void rseq_prepare_unload(void)
+static inline __attribute__((always_inline))
+void rseq_prepare_unload(void)
{
rseq_clear_rseq_cs();
}
+/*
+ * Refer to rseq/pseudocode.h for documentation and pseudo-code of the
+ * rseq critical section helpers.
+ */
+#include "rseq/pseudocode.h"
+
+static inline __attribute__((always_inline))
+int rseq_load_cbne_store__ptr(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
+ intptr_t *v, intptr_t expect,
+ intptr_t newv, int cpu)
+{
+ if (rseq_mo != RSEQ_MO_RELAXED)
+ return -1;
+ switch (percpu_mode) {
+ case RSEQ_PERCPU_CPU_ID:
+ return rseq_load_cbne_store__ptr_relaxed_cpu_id(v, expect, newv, cpu);
+ case RSEQ_PERCPU_MM_CID:
+ return rseq_load_cbne_store__ptr_relaxed_mm_cid(v, expect, newv, cpu);
+ default:
+ return -1;
+ }
+}
+
+static inline __attribute__((always_inline))
+int rseq_load_cbeq_store_add_load_store__ptr(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
+ intptr_t *v, intptr_t expectnot, long voffp, intptr_t *load,
+ int cpu)
+{
+ if (rseq_mo != RSEQ_MO_RELAXED)
+ return -1;
+ switch (percpu_mode) {
+ case RSEQ_PERCPU_CPU_ID:
+ return rseq_load_cbeq_store_add_load_store__ptr_relaxed_cpu_id(v, expectnot, voffp, load, cpu);
+ case RSEQ_PERCPU_MM_CID:
+ return rseq_load_cbeq_store_add_load_store__ptr_relaxed_mm_cid(v, expectnot, voffp, load, cpu);
+ default:
+ return -1;
+ }
+}
+
+static inline __attribute__((always_inline))
+int rseq_load_add_store__ptr(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
+ intptr_t *v, intptr_t count, int cpu)
+{
+ if (rseq_mo != RSEQ_MO_RELAXED)
+ return -1;
+ switch (percpu_mode) {
+ case RSEQ_PERCPU_CPU_ID:
+ return rseq_load_add_store__ptr_relaxed_cpu_id(v, count, cpu);
+ case RSEQ_PERCPU_MM_CID:
+ return rseq_load_add_store__ptr_relaxed_mm_cid(v, count, cpu);
+ default:
+ return -1;
+ }
+}
+
+#ifdef rseq_arch_has_load_add_load_load_add_store
+static inline __attribute__((always_inline))
+int rseq_load_add_load_load_add_store__ptr(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
+ intptr_t *ptr, long off, intptr_t inc, int cpu)
+{
+ if (rseq_mo != RSEQ_MO_RELAXED)
+ return -1;
+ switch (percpu_mode) {
+ case RSEQ_PERCPU_CPU_ID:
+ return rseq_load_add_load_load_add_store__ptr_relaxed_cpu_id(ptr, off, inc, cpu);
+ case RSEQ_PERCPU_MM_CID:
+ return rseq_load_add_load_load_add_store__ptr_relaxed_mm_cid(ptr, off, inc, cpu);
+ default:
+ return -1;
+ }
+}
+#endif
+
+static inline __attribute__((always_inline))
+int rseq_load_cbne_store_store__ptr(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
+ intptr_t *v, intptr_t expect,
+ intptr_t *v2, intptr_t newv2,
+ intptr_t newv, int cpu)
+{
+ switch (rseq_mo) {
+ case RSEQ_MO_RELAXED:
+ switch (percpu_mode) {
+ case RSEQ_PERCPU_CPU_ID:
+ return rseq_load_cbne_store_store__ptr_relaxed_cpu_id(v, expect, v2, newv2, newv, cpu);
+ case RSEQ_PERCPU_MM_CID:
+ return rseq_load_cbne_store_store__ptr_relaxed_mm_cid(v, expect, v2, newv2, newv, cpu);
+ default:
+ return -1;
+ }
+ case RSEQ_MO_RELEASE:
+ switch (percpu_mode) {
+ case RSEQ_PERCPU_CPU_ID:
+ return rseq_load_cbne_store_store__ptr_release_cpu_id(v, expect, v2, newv2, newv, cpu);
+ case RSEQ_PERCPU_MM_CID:
+ return rseq_load_cbne_store_store__ptr_release_mm_cid(v, expect, v2, newv2, newv, cpu);
+ default:
+ return -1;
+ }
+ case RSEQ_MO_ACQUIRE: /* Fallthrough */
+ case RSEQ_MO_ACQ_REL: /* Fallthrough */
+ case RSEQ_MO_CONSUME: /* Fallthrough */
+ case RSEQ_MO_SEQ_CST: /* Fallthrough */
+ default:
+ return -1;
+ }
+}
+
+static inline __attribute__((always_inline))
+int rseq_load_cbne_load_cbne_store__ptr(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
+ intptr_t *v, intptr_t expect,
+ intptr_t *v2, intptr_t expect2,
+ intptr_t newv, int cpu)
+{
+ if (rseq_mo != RSEQ_MO_RELAXED)
+ return -1;
+ switch (percpu_mode) {
+ case RSEQ_PERCPU_CPU_ID:
+ return rseq_load_cbne_load_cbne_store__ptr_relaxed_cpu_id(v, expect, v2, expect2, newv, cpu);
+ case RSEQ_PERCPU_MM_CID:
+ return rseq_load_cbne_load_cbne_store__ptr_relaxed_mm_cid(v, expect, v2, expect2, newv, cpu);
+ default:
+ return -1;
+ }
+}
+
+static inline __attribute__((always_inline))
+int rseq_load_cbne_memcpy_store__ptr(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
+ intptr_t *v, intptr_t expect,
+ void *dst, void *src, size_t len,
+ intptr_t newv, int cpu)
+{
+ switch (rseq_mo) {
+ case RSEQ_MO_RELAXED:
+ switch (percpu_mode) {
+ case RSEQ_PERCPU_CPU_ID:
+ return rseq_load_cbne_memcpy_store__ptr_relaxed_cpu_id(v, expect, dst, src, len, newv, cpu);
+ case RSEQ_PERCPU_MM_CID:
+ return rseq_load_cbne_memcpy_store__ptr_relaxed_mm_cid(v, expect, dst, src, len, newv, cpu);
+ default:
+ return -1;
+ }
+ case RSEQ_MO_RELEASE:
+ switch (percpu_mode) {
+ case RSEQ_PERCPU_CPU_ID:
+ return rseq_load_cbne_memcpy_store__ptr_release_cpu_id(v, expect, dst, src, len, newv, cpu);
+ case RSEQ_PERCPU_MM_CID:
+ return rseq_load_cbne_memcpy_store__ptr_release_mm_cid(v, expect, dst, src, len, newv, cpu);
+ default:
+ return -1;
+ }
+ case RSEQ_MO_ACQUIRE: /* Fallthrough */
+ case RSEQ_MO_ACQ_REL: /* Fallthrough */
+ case RSEQ_MO_CONSUME: /* Fallthrough */
+ case RSEQ_MO_SEQ_CST: /* Fallthrough */
+ default:
+ return -1;
+ }
+}
+
#ifdef __cplusplus
}
#endif
-#endif /* RSEQ_H_ */
+#endif /* _RSEQ_RSEQ_H */
projects / librseq.git / blobdiff