/* SPDX-FileCopyrightText: 2016-2022 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
/*
- * rseq.h
+ * rseq/rseq.h
*/
-#ifndef RSEQ_H
-#define RSEQ_H
+#ifndef _RSEQ_RSEQ_H
+#define _RSEQ_RSEQ_H
#include <stdint.h>
#include <stdbool.h>
#include <stdlib.h>
#include <stddef.h>
#include <assert.h>
-#include <rseq/rseq-abi.h>
-#include <rseq/compiler.h>
-
-#ifndef rseq_sizeof_field
-#define rseq_sizeof_field(TYPE, MEMBER) sizeof((((TYPE *)0)->MEMBER))
-#endif
-
-#ifndef rseq_offsetofend
-#define rseq_offsetofend(TYPE, MEMBER) \
- (offsetof(TYPE, MEMBER) + rseq_sizeof_field(TYPE, MEMBER))
-#endif
-
-/*
- * Empty code injection macros, override when testing.
- * It is important to consider that the ASM injection macros need to be
- * fully reentrant (e.g. do not modify the stack).
- */
-#ifndef RSEQ_INJECT_ASM
-#define RSEQ_INJECT_ASM(n)
-#endif
-#ifndef RSEQ_INJECT_C
-#define RSEQ_INJECT_C(n)
-#endif
+#include <rseq/abi.h>
+#include <rseq/compiler.h>
+#include <rseq/inject.h>
+#include <rseq/thread-pointer.h>
+#include <rseq/utils.h>
-#ifndef RSEQ_INJECT_INPUT
-#define RSEQ_INJECT_INPUT
-#endif
+enum rseq_mo {
+ RSEQ_MO_RELAXED = 0,
+ RSEQ_MO_CONSUME = 1, /* Unused */
+ RSEQ_MO_ACQUIRE = 2, /* Unused */
+ RSEQ_MO_RELEASE = 3,
+ RSEQ_MO_ACQ_REL = 4, /* Unused */
+ RSEQ_MO_SEQ_CST = 5, /* Unused */
+};
-#ifndef RSEQ_INJECT_CLOBBER
-#define RSEQ_INJECT_CLOBBER
-#endif
+enum rseq_percpu_mode {
+ RSEQ_PERCPU_CPU_ID = 0,
+ RSEQ_PERCPU_MM_CID = 1,
+};
-#ifndef RSEQ_INJECT_FAILED
-#define RSEQ_INJECT_FAILED
-#endif
+enum rseq_available_query {
+ RSEQ_AVAILABLE_QUERY_KERNEL = 0,
+ RSEQ_AVAILABLE_QUERY_LIBC = 1,
+};
/*
* User code can define RSEQ_GET_ABI_OVERRIDE to override the
*/
#ifndef RSEQ_GET_ABI_OVERRIDE
-# include <rseq/rseq-thread-pointer.h>
-
# ifdef __cplusplus
extern "C" {
# endif
*/
extern unsigned int rseq_feature_size;
-static inline struct rseq_abi *rseq_get_abi(void)
+/*
+ * Returns a pointer to the rseq area.
+ */
+static inline __attribute__((always_inline))
+struct rseq_abi *rseq_get_abi(void)
{
return (struct rseq_abi *) ((uintptr_t) rseq_thread_pointer() + rseq_offset);
}
#endif /* RSEQ_GET_ABI_OVERRIDE */
-enum rseq_mo {
- RSEQ_MO_RELAXED = 0,
- RSEQ_MO_CONSUME = 1, /* Unused */
- RSEQ_MO_ACQUIRE = 2, /* Unused */
- RSEQ_MO_RELEASE = 3,
- RSEQ_MO_ACQ_REL = 4, /* Unused */
- RSEQ_MO_SEQ_CST = 5, /* Unused */
-};
-enum rseq_percpu_mode {
- RSEQ_PERCPU_CPU_ID = 0,
- RSEQ_PERCPU_MM_CID = 1,
-};
+/*
+ * Architecture specific.
+ */
+#include <rseq/arch.h>
-#define rseq_likely(x) __builtin_expect(!!(x), 1)
-#define rseq_unlikely(x) __builtin_expect(!!(x), 0)
-#define rseq_barrier() __asm__ __volatile__("" : : : "memory")
-
-#define RSEQ_ACCESS_ONCE(x) (*(__volatile__ __typeof__(x) *)&(x))
-#define RSEQ_WRITE_ONCE(x, v) __extension__ ({ RSEQ_ACCESS_ONCE(x) = (v); })
-#define RSEQ_READ_ONCE(x) RSEQ_ACCESS_ONCE(x)
-
-#define __rseq_str_1(x) #x
-#define __rseq_str(x) __rseq_str_1(x)
-
-#define rseq_log(fmt, ...) \
- fprintf(stderr, fmt "(in %s() at " __FILE__ ":" __rseq_str(__LINE__)"\n", \
- ## __VA_ARGS__, __func__)
-
-#define rseq_bug(fmt, ...) \
- do { \
- rseq_log(fmt, ## __VA_ARGS__); \
- abort(); \
- } while (0)
-
-#if defined(__x86_64__) || defined(__i386__)
-#include <rseq/rseq-x86.h>
-#elif defined(__ARMEL__) || defined(__ARMEB__)
-#include <rseq/rseq-arm.h>
-#elif defined (__AARCH64EL__)
-#include <rseq/rseq-arm64.h>
-#elif defined(__PPC__)
-#include <rseq/rseq-ppc.h>
-#elif defined(__mips__)
-#include <rseq/rseq-mips.h>
-#elif defined(__s390__)
-#include <rseq/rseq-s390.h>
-#elif defined(__riscv)
-#include <rseq/rseq-riscv.h>
-#else
-#error unsupported target
-#endif
#ifdef __cplusplus
extern "C" {
*/
int32_t rseq_fallback_current_node(void);
-enum rseq_available_query {
- RSEQ_AVAILABLE_QUERY_KERNEL = 0,
- RSEQ_AVAILABLE_QUERY_LIBC = 1,
-};
-
/*
* Returns true if rseq is supported.
*/
* Values returned can be either the current CPU number, -1 (rseq is
* uninitialized), or -2 (rseq initialization has failed).
*/
-static inline int32_t rseq_current_cpu_raw(void)
+static inline __attribute__((always_inline))
+int32_t rseq_current_cpu_raw(void)
{
return RSEQ_READ_ONCE(rseq_get_abi()->cpu_id);
}
* return value of rseq_current_cpu_raw() if the rseq asm sequence
* does not need to be invoked.
*/
-static inline uint32_t rseq_cpu_start(void)
+static inline __attribute__((always_inline))
+uint32_t rseq_cpu_start(void)
{
return RSEQ_READ_ONCE(rseq_get_abi()->cpu_id_start);
}
-static inline uint32_t rseq_current_cpu(void)
+static inline __attribute__((always_inline))
+uint32_t rseq_current_cpu(void)
{
int32_t cpu;
return cpu;
}
-static inline bool rseq_node_id_available(void)
+static inline __attribute__((always_inline))
+bool rseq_node_id_available(void)
{
return (int) rseq_feature_size >= (int) rseq_offsetofend(struct rseq_abi, node_id);
}
/*
* Current NUMA node number.
*/
-static inline uint32_t rseq_current_node_id(void)
+static inline __attribute__((always_inline))
+uint32_t rseq_current_node_id(void)
{
assert(rseq_node_id_available());
return RSEQ_READ_ONCE(rseq_get_abi()->node_id);
}
-static inline bool rseq_mm_cid_available(void)
+static inline __attribute__((always_inline))
+bool rseq_mm_cid_available(void)
{
return (int) rseq_feature_size >= (int) rseq_offsetofend(struct rseq_abi, mm_cid);
}
-static inline uint32_t rseq_current_mm_cid(void)
+static inline __attribute__((always_inline))
+uint32_t rseq_current_mm_cid(void)
{
return RSEQ_READ_ONCE(rseq_get_abi()->mm_cid);
}
-static inline void rseq_clear_rseq_cs(void)
+static inline __attribute__((always_inline))
+void rseq_clear_rseq_cs(void)
{
RSEQ_WRITE_ONCE(rseq_get_abi()->rseq_cs.arch.ptr, 0);
}
* holding the struct rseq_cs or reclaim of the code pointed to by struct
* rseq_cs start_ip and post_commit_offset fields.
*/
-static inline void rseq_prepare_unload(void)
+static inline __attribute__((always_inline))
+void rseq_prepare_unload(void)
{
rseq_clear_rseq_cs();
}
/*
- * Refer to rseq-pseudocode.h for documentation and pseudo-code of the
+ * Refer to rseq/pseudocode.h for documentation and pseudo-code of the
* rseq critical section helpers.
*/
-#include "rseq-pseudocode.h"
+#include "rseq/pseudocode.h"
static inline __attribute__((always_inline))
int rseq_load_cbne_store__ptr(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
}
}
-#ifdef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
+#ifdef rseq_arch_has_load_add_load_load_add_store
static inline __attribute__((always_inline))
-int rseq_load_add_load_add_store__ptr(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
+int rseq_load_add_load_load_add_store__ptr(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
intptr_t *ptr, long off, intptr_t inc, int cpu)
{
if (rseq_mo != RSEQ_MO_RELAXED)
return -1;
switch (percpu_mode) {
case RSEQ_PERCPU_CPU_ID:
- return rseq_load_add_load_add_store__ptr_relaxed_cpu_id(ptr, off, inc, cpu);
+ return rseq_load_add_load_load_add_store__ptr_relaxed_cpu_id(ptr, off, inc, cpu);
case RSEQ_PERCPU_MM_CID:
- return rseq_load_add_load_add_store__ptr_relaxed_mm_cid(ptr, off, inc, cpu);
+ return rseq_load_add_load_load_add_store__ptr_relaxed_mm_cid(ptr, off, inc, cpu);
default:
return -1;
}
}
#endif
-#endif /* RSEQ_H_ */
+#endif /* _RSEQ_RSEQ_H */
projects / librseq.git / blobdiff