From 2d5330939d35d566f9d246758d3bcc340b7dd038 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Sun, 23 Jan 2022 14:07:54 -0500 Subject: [PATCH] Introduce rseq-abi.h Introduce our own header for the rseq kernel ABI, so we do not depend on Linux kernel headers. Use this header's rseq_cs.arch.ptr field as accessor for the rseq_cs ptr on 32-bit. Remove the extra/ directory because the copy of the Linux kernel headers is not needed anymore. Signed-off-by: Mathieu Desnoyers Change-Id: I0170f2bcaa7defd3a0d3f9a38115bc7d321ddde2 --- extra/README | 9 --- include/Makefile.am | 1 + extra/linux/rseq.h => include/rseq/rseq-abi.h | 80 ++++++++++--------- include/rseq/rseq-arm.h | 18 ++--- include/rseq/rseq-arm64.h | 18 ++--- include/rseq/rseq-mips.h | 18 ++--- include/rseq/rseq-ppc.h | 20 ++--- include/rseq/rseq-s390.h | 16 ++-- include/rseq/rseq-x86.h | 2 +- include/rseq/rseq.h | 28 ++----- src/rseq.c | 12 +-- 11 files changed, 94 insertions(+), 128 deletions(-) delete mode 100644 extra/README rename extra/linux/rseq.h => include/rseq/rseq-abi.h (74%) diff --git a/extra/README b/extra/README deleted file mode 100644 index f34f3b8..0000000 --- a/extra/README +++ /dev/null @@ -1,9 +0,0 @@ -Kernel headers for test builds. Do not use for production builds. - -- linux/rseq.h - - It is a copy from the upstream Linux v5.0 kernel uapi. - - It also contains definitive system call number definitions for the rseq - system call, in case the kernel uapi headers are not recent enough - to define them. diff --git a/include/Makefile.am b/include/Makefile.am index ec68ae0..8e37813 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -6,6 +6,7 @@ nobase_include_HEADERS = \ rseq/compiler.h \ rseq/rseq.h \ + rseq/rseq-abi.h \ rseq/rseq-arm.h \ rseq/rseq-mips.h \ rseq/rseq-ppc.h \ diff --git a/extra/linux/rseq.h b/include/rseq/rseq-abi.h similarity index 74% rename from extra/linux/rseq.h rename to include/rseq/rseq-abi.h index 306a0ba..4102d6e 100644 --- a/extra/linux/rseq.h +++ b/include/rseq/rseq-abi.h @@ -1,9 +1,9 @@ /* SPDX-License-Identifier: GPL-2.0+ WITH Linux-syscall-note */ -#ifndef _UAPI_LINUX_RSEQ_H -#define _UAPI_LINUX_RSEQ_H +#ifndef _RSEQ_ABI_H +#define _RSEQ_ABI_H /* - * linux/rseq.h + * rseq/rseq-abi.h * * Restartable sequences system call API * @@ -13,39 +13,39 @@ #include #include -enum rseq_cpu_id_state { - RSEQ_CPU_ID_UNINITIALIZED = -1, - RSEQ_CPU_ID_REGISTRATION_FAILED = -2, +enum rseq_abi_cpu_id_state { + RSEQ_ABI_CPU_ID_UNINITIALIZED = -1, + RSEQ_ABI_CPU_ID_REGISTRATION_FAILED = -2, }; -enum rseq_flags { - RSEQ_FLAG_UNREGISTER = (1 << 0), +enum rseq_abi_flags { + RSEQ_ABI_FLAG_UNREGISTER = (1 << 0), }; -enum rseq_cs_flags_bit { - RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT = 0, - RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT = 1, - RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT = 2, +enum rseq_abi_cs_flags_bit { + RSEQ_ABI_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT = 0, + RSEQ_ABI_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT = 1, + RSEQ_ABI_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT = 2, }; -enum rseq_cs_flags { - RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT = - (1U << RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT), - RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL = - (1U << RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT), - RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE = - (1U << RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT), +enum rseq_abi_cs_flags { + RSEQ_ABI_CS_FLAG_NO_RESTART_ON_PREEMPT = + (1U << RSEQ_ABI_CS_FLAG_NO_RESTART_ON_PREEMPT_BIT), + RSEQ_ABI_CS_FLAG_NO_RESTART_ON_SIGNAL = + (1U << RSEQ_ABI_CS_FLAG_NO_RESTART_ON_SIGNAL_BIT), + RSEQ_ABI_CS_FLAG_NO_RESTART_ON_MIGRATE = + (1U << RSEQ_ABI_CS_FLAG_NO_RESTART_ON_MIGRATE_BIT), }; /* - * struct rseq_cs is aligned on 4 * 8 bytes to ensure it is always + * struct rseq_abi_cs is aligned on 4 * 8 bytes to ensure it is always * contained within a single cache-line. It is usually declared as * link-time constant data. */ -struct rseq_cs { +struct rseq_abi_cs { /* Version of this structure. */ __u32 version; - /* enum rseq_cs_flags */ + /* enum rseq_abi_cs_flags */ __u32 flags; __u64 start_ip; /* Offset from start_ip. */ @@ -54,12 +54,12 @@ struct rseq_cs { } __attribute__((aligned(4 * sizeof(__u64)))); /* - * struct rseq is aligned on 4 * 8 bytes to ensure it is always + * struct rseq_abi is aligned on 4 * 8 bytes to ensure it is always * contained within a single cache-line. * - * A single struct rseq per thread is allowed. + * A single struct rseq_abi per thread is allowed. */ -struct rseq { +struct rseq_abi { /* * Restartable sequences cpu_id_start field. Updated by the * kernel. Read by user-space with single-copy atomicity @@ -108,19 +108,23 @@ struct rseq { */ union { __u64 ptr64; -#ifdef __LP64__ - __u64 ptr; -#else + + /* + * The "arch" field provides architecture accessor for + * the ptr field based on architecture pointer size and + * endianness. + */ struct { -#if (defined(__BYTE_ORDER) && (__BYTE_ORDER == __BIG_ENDIAN)) || defined(__BIG_ENDIAN) +#ifdef __LP64__ + __u64 ptr; +#elif defined(__BYTE_ORDER) ? (__BYTE_ORDER == __BIG_ENDIAN) : defined(__BIG_ENDIAN) __u32 padding; /* Initialized to zero. */ - __u32 ptr32; -#else /* LITTLE */ - __u32 ptr32; + __u32 ptr; +#else + __u32 ptr; __u32 padding; /* Initialized to zero. */ -#endif /* ENDIAN */ - } ptr; #endif + } arch; } rseq_cs; /* @@ -131,13 +135,13 @@ struct rseq { * Mainly used for single-stepping through rseq critical sections * with debuggers. * - * - RSEQ_CS_FLAG_NO_RESTART_ON_PREEMPT + * - RSEQ_ABI_CS_FLAG_NO_RESTART_ON_PREEMPT * Inhibit instruction sequence block restart on preemption * for this thread. - * - RSEQ_CS_FLAG_NO_RESTART_ON_SIGNAL + * - RSEQ_ABI_CS_FLAG_NO_RESTART_ON_SIGNAL * Inhibit instruction sequence block restart on signal * delivery for this thread. - * - RSEQ_CS_FLAG_NO_RESTART_ON_MIGRATE + * - RSEQ_ABI_CS_FLAG_NO_RESTART_ON_MIGRATE * Inhibit instruction sequence block restart on migration for * this thread. */ @@ -186,4 +190,4 @@ struct rseq { #endif -#endif /* _UAPI_LINUX_RSEQ_H */ +#endif /* _RSEQ_ABI_H */ diff --git a/include/rseq/rseq-arm.h b/include/rseq/rseq-arm.h index 975c2a4..9645d82 100644 --- a/include/rseq/rseq-arm.h +++ b/include/rseq/rseq-arm.h @@ -60,8 +60,6 @@ #define RSEQ_SIG 0xe7f5def3 /* udf #24035 ; 0x5de3 */ #endif -#define RSEQ_CS_PTR RSEQ_CS_PTR32 - #define rseq_smp_mb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") #define rseq_smp_rmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") #define rseq_smp_wmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc") @@ -185,7 +183,7 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), [v] "m" (*v), [expect] "r" (expect), [newv] "r" (newv) @@ -256,7 +254,7 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* final store input */ [v] "m" (*v), [expectnot] "r" (expectnot), @@ -318,7 +316,7 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), [v] "m" (*v), [count] "Ir" (count) RSEQ_INJECT_INPUT @@ -383,7 +381,7 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* try store input */ [v2] "m" (*v2), [newv2] "r" (newv2), @@ -460,7 +458,7 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* try store input */ [v2] "m" (*v2), [newv2] "r" (newv2), @@ -541,7 +539,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* cmp2 input */ [v2] "m" (*v2), [expect2] "r" (expect2), @@ -663,7 +661,7 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* final store input */ [v] "m" (*v), [expect] "r" (expect), @@ -787,7 +785,7 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* final store input */ [v] "m" (*v), [expect] "r" (expect), diff --git a/include/rseq/rseq-arm64.h b/include/rseq/rseq-arm64.h index b0e6020..4511bec 100644 --- a/include/rseq/rseq-arm64.h +++ b/include/rseq/rseq-arm64.h @@ -21,8 +21,6 @@ #define RSEQ_SIG RSEQ_SIG_DATA -#define RSEQ_CS_PTR rseq_cs.ptr - #define rseq_smp_mb() __asm__ __volatile__ ("dmb ish" ::: "memory") #define rseq_smp_rmb() __asm__ __volatile__ ("dmb ishld" ::: "memory") #define rseq_smp_wmb() __asm__ __volatile__ ("dmb ishst" ::: "memory") @@ -233,7 +231,7 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), [v] "Qo" (*v), [expect] "r" (expect), [newv] "r" (newv) @@ -294,7 +292,7 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), [v] "Qo" (*v), [expectnot] "r" (expectnot), [load] "Qo" (*load), @@ -349,7 +347,7 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), [v] "Qo" (*v), [count] "r" (count) RSEQ_INJECT_INPUT @@ -403,7 +401,7 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), [expect] "r" (expect), [v] "Qo" (*v), [newv] "r" (newv), @@ -466,7 +464,7 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), [expect] "r" (expect), [v] "Qo" (*v), [newv] "r" (newv), @@ -531,7 +529,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), [v] "Qo" (*v), [expect] "r" (expect), [v2] "Qo" (*v2), @@ -597,7 +595,7 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), [expect] "r" (expect), [v] "Qo" (*v), [newv] "r" (newv), @@ -661,7 +659,7 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), [expect] "r" (expect), [v] "Qo" (*v), [newv] "r" (newv), diff --git a/include/rseq/rseq-mips.h b/include/rseq/rseq-mips.h index cb63c55..67cf908 100644 --- a/include/rseq/rseq-mips.h +++ b/include/rseq/rseq-mips.h @@ -71,7 +71,6 @@ do { \ # define LONG_S "sd" # define LONG_ADDI "daddiu" # define U32_U64_PAD(x) x -# define RSEQ_CS_PTR rseq_cs.ptr #elif _MIPS_SZLONG == 32 # define LONG ".word" # define LONG_LA "la" @@ -83,7 +82,6 @@ do { \ # else # define U32_U64_PAD(x) x ", 0x0" # endif -# define RSEQ_CS_PTR RSEQ_CS_PTR32 #else # error unsupported _MIPS_SZLONG #endif @@ -190,7 +188,7 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), [v] "m" (*v), [expect] "r" (expect), [newv] "r" (newv) @@ -254,7 +252,7 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* final store input */ [v] "m" (*v), [expectnot] "r" (expectnot), @@ -311,7 +309,7 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), [v] "m" (*v), [count] "Ir" (count) RSEQ_INJECT_INPUT @@ -371,7 +369,7 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* try store input */ [v2] "m" (*v2), [newv2] "r" (newv2), @@ -441,7 +439,7 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* try store input */ [v2] "m" (*v2), [newv2] "r" (newv2), @@ -513,7 +511,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* cmp2 input */ [v2] "m" (*v2), [expect2] "r" (expect2), @@ -626,7 +624,7 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* final store input */ [v] "m" (*v), [expect] "r" (expect), @@ -742,7 +740,7 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* final store input */ [v] "m" (*v), [expect] "r" (expect), diff --git a/include/rseq/rseq-ppc.h b/include/rseq/rseq-ppc.h index 22ca2e2..10c92a3 100644 --- a/include/rseq/rseq-ppc.h +++ b/include/rseq/rseq-ppc.h @@ -55,8 +55,6 @@ do { \ #define RSEQ_CMP_LONG "cmpd " #define RSEQ_CMP_LONG_INT "cmpdi " -#define RSEQ_CS_PTR rseq_cs.ptr - #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ start_ip, post_commit_offset, abort_ip) \ ".pushsection __rseq_cs, \"aw\"\n\t" \ @@ -102,8 +100,6 @@ do { \ #define RSEQ_CMP_LONG "cmpw " #define RSEQ_CMP_LONG_INT "cmpwi " -#define RSEQ_CS_PTR RSEQ_CS_PTR32 - #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ start_ip, post_commit_offset, abort_ip) \ ".pushsection __rseq_cs, \"aw\"\n\t" \ @@ -246,7 +242,7 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), [v] "m" (*v), [expect] "r" (expect), [newv] "r" (newv) @@ -317,7 +313,7 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* final store input */ [v] "m" (*v), [expectnot] "r" (expectnot), @@ -380,7 +376,7 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* final store input */ [v] "m" (*v), [count] "r" (count) @@ -443,7 +439,7 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* try store input */ [v2] "m" (*v2), [newv2] "r" (newv2), @@ -518,7 +514,7 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* try store input */ [v2] "m" (*v2), [newv2] "r" (newv2), @@ -594,7 +590,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* cmp2 input */ [v2] "m" (*v2), [expect2] "r" (expect2), @@ -675,7 +671,7 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* final store input */ [v] "m" (*v), [expect] "r" (expect), @@ -756,7 +752,7 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* final store input */ [v] "m" (*v), [expect] "r" (expect), diff --git a/include/rseq/rseq-s390.h b/include/rseq/rseq-s390.h index 94eb55f..7b1e351 100644 --- a/include/rseq/rseq-s390.h +++ b/include/rseq/rseq-s390.h @@ -34,8 +34,6 @@ do { \ #ifdef __s390x__ -#define RSEQ_CS_PTR rseq_cs.ptr - #define LONG_L "lg" #define LONG_S "stg" #define LONG_LT_R "ltgr" @@ -71,8 +69,6 @@ do { \ #elif __s390__ -#define RSEQ_CS_PTR RSEQ_CS_PTR32 - #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \ start_ip, post_commit_offset, abort_ip) \ ".pushsection __rseq_cs, \"aw\"\n\t" \ @@ -170,7 +166,7 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), [v] "m" (*v), [expect] "r" (expect), [newv] "r" (newv) @@ -243,7 +239,7 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* final store input */ [v] "m" (*v), [expectnot] "r" (expectnot), @@ -303,7 +299,7 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* final store input */ [v] "m" (*v), [count] "r" (count) @@ -365,7 +361,7 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* try store input */ [v2] "m" (*v2), [newv2] "r" (newv2), @@ -449,7 +445,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* cmp2 input */ [v2] "m" (*v2), [expect2] "r" (expect2), @@ -563,7 +559,7 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->cpu_id), - [rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR), + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* final store input */ [v] "m" (*v), [expect] "r" (expect), diff --git a/include/rseq/rseq-x86.h b/include/rseq/rseq-x86.h index 7e0417f..0bd472d 100644 --- a/include/rseq/rseq-x86.h +++ b/include/rseq/rseq-x86.h @@ -23,7 +23,7 @@ * (TODO: revisit after migration to glibc's ABI) */ -/* Offset of cpu_id and rseq_cs fields in struct rseq. */ +/* Offset of cpu_id and rseq_cs fields in struct rseq_abi. */ #define RSEQ_CPU_ID_OFFSET 4 #define RSEQ_CS_OFFSET 8 diff --git a/include/rseq/rseq.h b/include/rseq/rseq.h index 061e82a..8e58139 100644 --- a/include/rseq/rseq.h +++ b/include/rseq/rseq.h @@ -17,21 +17,9 @@ #include #include #include -#include +#include #include -/* Work-around rseq.h uapi bug wrt endianness on little endian. */ -#if !(defined(__BYTE_ORDER) ? __BYTE_ORDER == __BIG_ENDIAN : defined(__BIG_ENDIAN)) -# if (defined(__BYTE_ORDER) && (__BYTE_ORDER == __BIG_ENDIAN)) || defined(__BIG_ENDIAN) -/* Kernel uapi mistakenly orders padding vs ptr32. */ -# define RSEQ_CS_PTR32 rseq_cs.ptr.padding -# endif -#endif - -#ifndef RSEQ_CS_PTR32 -# define RSEQ_CS_PTR32 rseq_cs.ptr.ptr32 -#endif - /* * Empty code injection macros, override when testing. * It is important to consider that the ASM injection macros need to be @@ -79,9 +67,9 @@ extern unsigned int rseq_size; /* Flags used during rseq registration. */ extern unsigned int rseq_flags; -static inline struct rseq *rseq_get_abi(void) +static inline struct rseq_abi *rseq_get_abi(void) { - return (struct rseq *) ((uintptr_t) rseq_thread_pointer() + rseq_offset); + return (struct rseq_abi *) ((uintptr_t) rseq_thread_pointer() + rseq_offset); } # ifdef __cplusplus @@ -189,21 +177,17 @@ static inline uint32_t rseq_current_cpu(void) static inline void rseq_clear_rseq_cs(void) { -#ifdef __LP64__ - RSEQ_WRITE_ONCE(rseq_get_abi()->rseq_cs.ptr, 0); -#else - RSEQ_WRITE_ONCE(rseq_get_abi()->RSEQ_CS_PTR32, 0); -#endif + RSEQ_WRITE_ONCE(rseq_get_abi()->rseq_cs.arch.ptr, 0); } /* * rseq_prepare_unload() should be invoked by each thread executing a rseq * critical section at least once between their last critical section and * library unload of the library defining the rseq critical section - * (struct rseq_cs). This also applies to use of rseq in code generated by + * (struct rseq_ab_cs). This also applies to use of rseq in code generated by * JIT: rseq_prepare_unload() should be invoked at least once by each * thread executing a rseq critical section before reclaim of the memory - * holding the struct rseq_cs. + * holding the struct rseq_abi_cs. */ static inline void rseq_prepare_unload(void) { diff --git a/src/rseq.c b/src/rseq.c index 54e080f..a122da2 100644 --- a/src/rseq.c +++ b/src/rseq.c @@ -50,11 +50,11 @@ unsigned int rseq_flags; static int rseq_ownership; static -__thread struct rseq __rseq_abi __attribute__((tls_model("initial-exec"))) = { - .cpu_id = RSEQ_CPU_ID_UNINITIALIZED, +__thread struct rseq_abi __rseq_abi __attribute__((tls_model("initial-exec"))) = { + .cpu_id = RSEQ_ABI_CPU_ID_UNINITIALIZED, }; -static int sys_rseq(struct rseq *rseq_abi, uint32_t rseq_len, +static int sys_rseq(struct rseq_abi *rseq_abi, uint32_t rseq_len, int flags, uint32_t sig) { return syscall(__NR_rseq, rseq_abi, rseq_len, flags, sig); @@ -85,7 +85,7 @@ int rseq_register_current_thread(void) /* Treat libc's ownership as a successful registration. */ return 0; } - rc = sys_rseq(&__rseq_abi, sizeof(struct rseq), 0, RSEQ_SIG); + rc = sys_rseq(&__rseq_abi, sizeof(struct rseq_abi), 0, RSEQ_SIG); if (rc) return -1; assert(rseq_current_cpu_raw() >= 0); @@ -100,7 +100,7 @@ int rseq_unregister_current_thread(void) /* Treat libc's ownership as a successful unregistration. */ return 0; } - rc = sys_rseq(&__rseq_abi, sizeof(struct rseq), RSEQ_FLAG_UNREGISTER, RSEQ_SIG); + rc = sys_rseq(&__rseq_abi, sizeof(struct rseq_abi), RSEQ_ABI_FLAG_UNREGISTER, RSEQ_SIG); if (rc) return -1; return 0; @@ -123,7 +123,7 @@ void rseq_init(void) return; rseq_ownership = 1; rseq_offset = (void *)&__rseq_abi - rseq_thread_pointer(); - rseq_size = sizeof(struct rseq); + rseq_size = sizeof(struct rseq_abi); rseq_flags = 0; } -- 2.34.1