[librseq.git] / include / rseq / arch / x86.h

/* SPDX-License-Identifier: MIT */
/* SPDX-FileCopyrightText: 2016-2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */

/*
 * rseq/arch/x86.h
 */

#ifndef _RSEQ_RSEQ_H
#error "Never use <rseq/arch/x86.h> directly; include <rseq/rseq.h> instead."
#endif

#include <stdint.h>

/*
 * RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those
 * are not part of the public API.
 */

/*
 * RSEQ_SIG is used with the following reserved undefined instructions, which
 * trap in user-space:
 *
 * x86-32:    0f b9 3d 53 30 05 53      ud1    0x53053053,%edi
 * x86-64:    0f b9 3d 53 30 05 53      ud1    0x53053053(%rip),%edi
 */
#define RSEQ_SIG	0x53053053

/*
 * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
 * operands, we cannot use "m" input operands, and rather pass the __rseq_abi
 * address through a "r" input operand.
 */

/*
 * Offset of cpu_id, rseq_cs, and mm_cid fields in struct rseq. Those
 * are defined explicitly as macros to be used from assembly.
 */
#define RSEQ_ASM_CPU_ID_OFFSET		4
#define RSEQ_ASM_CS_OFFSET		8
#define RSEQ_ASM_MM_CID_OFFSET		24

/*
 * Refer to the Linux kernel memory model (LKMM) for documentation of
 * the memory barriers. Expect all x86 hardware to be x86-TSO (Total
 * Store Order).
 */

/* CPU memory barrier. */
#define rseq_smp_mb()	\
	__asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
/* CPU read memory barrier */
#define rseq_smp_rmb()	rseq_barrier()
/* CPU write memory barrier */
#define rseq_smp_wmb()	rseq_barrier()

/* Acquire: One-way permeable barrier. */
#define rseq_smp_load_acquire(p)					\
__extension__ ({							\
	rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p));	\
	rseq_barrier();							\
	____p1;								\
})

/* Acquire barrier after control dependency. */
#define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_rmb()

/* Release: One-way permeable barrier. */
#define rseq_smp_store_release(p, v)					\
do {									\
	rseq_barrier();							\
	RSEQ_WRITE_ONCE(*(p), v);					\
} while (0)

/* Segment selector for the thread pointer. */
#ifdef RSEQ_ARCH_AMD64
# define RSEQ_ASM_TP_SEGMENT		%%fs
#else
# define RSEQ_ASM_TP_SEGMENT		%%gs
#endif

/*
 * Helper macro to define a variable of pointer type stored in a 64-bit
 * integer. Only used internally in rseq headers.
 */
#ifdef RSEQ_ARCH_AMD64
# define RSEQ_ASM_U64_PTR(x)		".quad " x
#else
# define RSEQ_ASM_U64_PTR(x)		".long " x ", 0x0"
#endif

#define RSEQ_ASM_U32(x)			".long " x

/*
 * Store the address of the critical section descriptor structure at
 * @cs_label into the @rseq_cs pointer and emit the label @label, which
 * is the beginning of the sequence of consecutive assembly instructions.
 *
 *  @label:
 *    Local label to the beginning of the sequence of consecutive assembly
 *    instructions.
 *  @cs_label:
 *    Source local label to the critical section descriptor structure.
 *  @rseq_cs:
 *    Destination pointer where to store the address of the critical
 *    section descriptor structure.
 */
#ifdef RSEQ_ARCH_AMD64
#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)		\
		RSEQ_INJECT_ASM(1)					\
		"leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t"	\
		"movq %%rax, " __rseq_str(rseq_cs) "\n\t"		\
		__rseq_str(label) ":\n\t"
#else
# define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)		\
		RSEQ_INJECT_ASM(1)					\
		"movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t"	\
		__rseq_str(label) ":\n\t"
#endif

/* Only used in RSEQ_ASM_DEFINE_TABLE. */
#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,			\
				start_ip, post_commit_offset, abort_ip)	\
		".pushsection __rseq_cs, \"aw\"\n\t"			\
		".balign 32\n\t"					\
		__rseq_str(label) ":\n\t"				\
		RSEQ_ASM_U32(__rseq_str(version)) "\n\t" 		\
		RSEQ_ASM_U32(__rseq_str(flags)) "\n\t" 			\
		RSEQ_ASM_U64_PTR(__rseq_str(start_ip)) "\n\t"		\
		RSEQ_ASM_U64_PTR(__rseq_str(post_commit_offset)) "\n\t" \
		RSEQ_ASM_U64_PTR(__rseq_str(abort_ip)) "\n\t"		\
		".popsection\n\t"					\
		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
		RSEQ_ASM_U64_PTR(__rseq_str(label) "b") "\n\t"		\
		".popsection\n\t"

/*
 * Define an rseq critical section structure of version 0 with no flags.
 *
 *  @label:
 *    Local label for the beginning of the critical section descriptor
 *    structure.
 *  @start_ip:
 *    Pointer to the first instruction of the sequence of consecutive assembly
 *    instructions.
 *  @post_commit_ip:
 *    Pointer to the instruction after the last instruction of the sequence of
 *    consecutive assembly instructions.
 *  @abort_ip:
 *    Pointer to the instruction where to move the execution flow in case of
 *    abort of the sequence of consecutive assembly instructions.
 */
#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,		\
				(post_commit_ip) - (start_ip), abort_ip)

/*
 * Define the @exit_ip pointer as an exit point for the sequence of consecutive
 * assembly instructions at @start_ip.
 *
 *  @start_ip:
 *    Pointer to the first instruction of the sequence of consecutive assembly
 *    instructions.
 *  @exit_ip:
 *    Pointer to an exit point instruction.
 *
 * Exit points of a rseq critical section consist of all instructions outside
 * of the critical section where a critical section can either branch to or
 * reach through the normal course of its execution. The abort IP and the
 * post-commit IP are already part of the __rseq_cs section and should not be
 * explicitly defined as additional exit points. Knowing all exit points is
 * useful to assist debuggers stepping over the critical section.
 */
#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
		RSEQ_ASM_U64_PTR(__rseq_str(start_ip)) "\n\t"		\
		RSEQ_ASM_U64_PTR(__rseq_str(exit_ip)) "\n\t"		\
		".popsection\n\t"

/*
 * Define a critical section abort handler.
 *
 *  @label:
 *    Local label to the abort handler.
 *  @teardown:
 *    Sequence of instructions to run on abort.
 *  @abort_label:
 *    C label to jump to at the end of the sequence.
 */
#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)		\
		".pushsection __rseq_failure, \"ax\"\n\t"		\
		/*							\
		 * Disassembler-friendly signature:			\
		 *   x86-32: ud1 <sig>,%edi				\
		 *   x86-64: ud1 <sig>(%rip),%edi			\
		 */							\
		".byte 0x0f, 0xb9, 0x3d\n\t"				\
		".long " __rseq_str(RSEQ_SIG) "\n\t"			\
		__rseq_str(label) ":\n\t"				\
		teardown						\
		"jmp %l[" __rseq_str(abort_label) "]\n\t"		\
		".popsection\n\t"

/*
 * Define a critical section teardown handler.
 *
 *  @label:
 *    Local label to the teardown handler.
 *  @teardown:
 *    Sequence of instructions to run on teardown.
 *  @target_label:
 *    C label to jump to at the end of the sequence.
 */
#define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label)		\
		".pushsection __rseq_failure, \"ax\"\n\t"		\
		__rseq_str(label) ":\n\t"				\
		teardown						\
		"jmp %l[" __rseq_str(target_label) "]\n\t"		\
		".popsection\n\t"

/* Jump to local label @label when @cpu_id != @current_cpu_id. */
#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label)		\
		RSEQ_INJECT_ASM(2)					\
		"cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
		"jnz " __rseq_str(label) "\n\t"

/* Per-cpu-id indexing. */

#define RSEQ_TEMPLATE_INDEX_CPU_ID
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq/arch/x86/bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED

#define RSEQ_TEMPLATE_MO_RELEASE
#include "rseq/arch/x86/bits.h"
#undef RSEQ_TEMPLATE_MO_RELEASE
#undef RSEQ_TEMPLATE_INDEX_CPU_ID

/* Per-mm-cid indexing. */

#define RSEQ_TEMPLATE_INDEX_MM_CID
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq/arch/x86/bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED

#define RSEQ_TEMPLATE_MO_RELEASE
#include "rseq/arch/x86/bits.h"
#undef RSEQ_TEMPLATE_MO_RELEASE
#undef RSEQ_TEMPLATE_INDEX_MM_CID

/* APIs which are not indexed. */

#define RSEQ_TEMPLATE_INDEX_NONE
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq/arch/x86/bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED
#undef RSEQ_TEMPLATE_INDEX_NONE
Commit	Line	Data
90702366	1	/* SPDX-License-Identifier: MIT */
abac6819	2	/* SPDX-FileCopyrightText: 2016-2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
f2d7b530	3
784b0012	4	/*
44ec21eb	5	* rseq/arch/x86.h
784b0012 MD	6	*/
784b0012 MD	7
44ec21eb MJ	8	#ifndef _RSEQ_RSEQ_H
44ec21eb MJ	9	#error "Never use <rseq/arch/x86.h> directly; include <rseq/rseq.h> instead."
809f5ee3 MD	10	#endif
809f5ee3 MD	11
784b0012 MD	12	#include <stdint.h>
784b0012 MD	13
abac6819 MD	14	/*
	15	* RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those
	16	* are not part of the public API.
	17	*/
	18
a5fe9e95 MD	19	/*
	20	* RSEQ_SIG is used with the following reserved undefined instructions, which
	21	* trap in user-space:
	22	*
	23	* x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi
	24	* x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi
	25	*/
784b0012 MD	26	#define RSEQ_SIG 0x53053053
784b0012 MD	27
809f5ee3 MD	28	/*
	29	* Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
	30	* operands, we cannot use "m" input operands, and rather pass the __rseq_abi
	31	* address through a "r" input operand.
	32	*/
	33
abac6819 MD	34	/*
	35	* Offset of cpu_id, rseq_cs, and mm_cid fields in struct rseq. Those
	36	* are defined explicitly as macros to be used from assembly.
	37	*/
d9c11a15 MD	38	#define RSEQ_ASM_CPU_ID_OFFSET 4
	39	#define RSEQ_ASM_CS_OFFSET 8
	40	#define RSEQ_ASM_MM_CID_OFFSET 24
05ebd3f9	41
abac6819 MD	42	/*
	43	* Refer to the Linux kernel memory model (LKMM) for documentation of
	44	* the memory barriers. Expect all x86 hardware to be x86-TSO (Total
	45	* Store Order).
	46	*/
	47
	48	/* CPU memory barrier. */
784b0012 MD	49	#define rseq_smp_mb() \
784b0012 MD	50	__asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
abac6819	51	/* CPU read memory barrier */
784b0012	52	#define rseq_smp_rmb() rseq_barrier()
abac6819	53	/* CPU write memory barrier */
784b0012 MD	54	#define rseq_smp_wmb() rseq_barrier()
784b0012 MD	55
abac6819	56	/* Acquire: One-way permeable barrier. */
784b0012 MD	57	#define rseq_smp_load_acquire(p) \
784b0012 MD	58	__extension__ ({ \
3664a718	59	rseq_unqual_scalar_typeof((p)) ____p1 = RSEQ_READ_ONCE((p)); \
784b0012 MD	60	rseq_barrier(); \
	61	____p1; \
	62	})
	63
abac6819	64	/* Acquire barrier after control dependency. */
784b0012 MD	65	#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
784b0012 MD	66
abac6819	67	/* Release: One-way permeable barrier. */
784b0012 MD	68	#define rseq_smp_store_release(p, v) \
	69	do { \
	70	rseq_barrier(); \
809f5ee3	71	RSEQ_WRITE_ONCE(*(p), v); \
784b0012 MD	72	} while (0)
784b0012 MD	73
abac6819	74	/* Segment selector for the thread pointer. */
b52031b2 MD	75	#ifdef RSEQ_ARCH_AMD64
	76	# define RSEQ_ASM_TP_SEGMENT %%fs
	77	#else
	78	# define RSEQ_ASM_TP_SEGMENT %%gs
	79	#endif
784b0012	80
90d9876e	81	/*
b52031b2 MD	82	* Helper macro to define a variable of pointer type stored in a 64-bit
b52031b2 MD	83	* integer. Only used internally in rseq headers.
90d9876e	84	*/
b52031b2 MD	85	#ifdef RSEQ_ARCH_AMD64
	86	# define RSEQ_ASM_U64_PTR(x) ".quad " x
	87	#else
	88	# define RSEQ_ASM_U64_PTR(x) ".long " x ", 0x0"
	89	#endif
90d9876e	90
78951921 MD	91	#define RSEQ_ASM_U32(x) ".long " x
78951921 MD	92
abac6819 MD	93	/*
	94	* Store the address of the critical section descriptor structure at
	95	* @cs_label into the @rseq_cs pointer and emit the label @label, which
	96	* is the beginning of the sequence of consecutive assembly instructions.
	97	*
	98	* @label:
	99	* Local label to the beginning of the sequence of consecutive assembly
	100	* instructions.
	101	* @cs_label:
	102	* Source local label to the critical section descriptor structure.
	103	* @rseq_cs:
	104	* Destination pointer where to store the address of the critical
	105	* section descriptor structure.
	106	*/
b52031b2	107	#ifdef RSEQ_ARCH_AMD64
784b0012 MD	108	#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
	109	RSEQ_INJECT_ASM(1) \
	110	"leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \
05ebd3f9	111	"movq %%rax, " __rseq_str(rseq_cs) "\n\t" \
784b0012	112	__rseq_str(label) ":\n\t"
b52031b2 MD	113	#else
	114	# define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
	115	RSEQ_INJECT_ASM(1) \
	116	"movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \
	117	__rseq_str(label) ":\n\t"
	118	#endif
53831176	119
abac6819	120	/* Only used in RSEQ_ASM_DEFINE_TABLE. */
784b0012 MD	121	#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
784b0012 MD	122	start_ip, post_commit_offset, abort_ip) \
dd01d0fb	123	".pushsection __rseq_cs, \"aw\"\n\t" \
784b0012 MD	124	".balign 32\n\t" \
784b0012 MD	125	__rseq_str(label) ":\n\t" \
78951921 MD	126	RSEQ_ASM_U32(__rseq_str(version)) "\n\t" \
78951921 MD	127	RSEQ_ASM_U32(__rseq_str(flags)) "\n\t" \
b52031b2 MD	128	RSEQ_ASM_U64_PTR(__rseq_str(start_ip)) "\n\t" \
	129	RSEQ_ASM_U64_PTR(__rseq_str(post_commit_offset)) "\n\t" \
	130	RSEQ_ASM_U64_PTR(__rseq_str(abort_ip)) "\n\t" \
dd01d0fb MD	131	".popsection\n\t" \
dd01d0fb MD	132	".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
b52031b2	133	RSEQ_ASM_U64_PTR(__rseq_str(label) "b") "\n\t" \
784b0012 MD	134	".popsection\n\t"
784b0012 MD	135
b52031b2 MD	136	/*
	137	* Define an rseq critical section structure of version 0 with no flags.
	138	*
	139	* @label:
	140	* Local label for the beginning of the critical section descriptor
	141	* structure.
	142	* @start_ip:
	143	* Pointer to the first instruction of the sequence of consecutive assembly
	144	* instructions.
	145	* @post_commit_ip:
	146	* Pointer to the instruction after the last instruction of the sequence of
	147	* consecutive assembly instructions.
	148	* @abort_ip:
	149	* Pointer to the instruction where to move the execution flow in case of
	150	* abort of the sequence of consecutive assembly instructions.
	151	*/
	152	#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
	153	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
	154	(post_commit_ip) - (start_ip), abort_ip)
	155
90d9876e	156	/*
abac6819 MD	157	* Define the @exit_ip pointer as an exit point for the sequence of consecutive
	158	* assembly instructions at @start_ip.
	159	*
	160	* @start_ip:
	161	* Pointer to the first instruction of the sequence of consecutive assembly
	162	* instructions.
	163	* @exit_ip:
	164	* Pointer to an exit point instruction.
	165	*
90d9876e	166	* Exit points of a rseq critical section consist of all instructions outside
fd622cad MD	167	* of the critical section where a critical section can either branch to or
	168	* reach through the normal course of its execution. The abort IP and the
	169	* post-commit IP are already part of the __rseq_cs section and should not be
	170	* explicitly defined as additional exit points. Knowing all exit points is
	171	* useful to assist debuggers stepping over the critical section.
90d9876e MD	172	*/
	173	#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
	174	".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
b52031b2 MD	175	RSEQ_ASM_U64_PTR(__rseq_str(start_ip)) "\n\t" \
b52031b2 MD	176	RSEQ_ASM_U64_PTR(__rseq_str(exit_ip)) "\n\t" \
90d9876e MD	177	".popsection\n\t"
90d9876e MD	178
abac6819 MD	179	/*
	180	* Define a critical section abort handler.
	181	*
	182	* @label:
	183	* Local label to the abort handler.
	184	* @teardown:
	185	* Sequence of instructions to run on abort.
	186	* @abort_label:
	187	* C label to jump to at the end of the sequence.
	188	*/
784b0012 MD	189	#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
784b0012 MD	190	".pushsection __rseq_failure, \"ax\"\n\t" \
abac6819 MD	191	/* \
	192	* Disassembler-friendly signature: \
	193	* x86-32: ud1 <sig>,%edi \
	194	* x86-64: ud1 <sig>(%rip),%edi \
	195	*/ \
a5fe9e95	196	".byte 0x0f, 0xb9, 0x3d\n\t" \
784b0012 MD	197	".long " __rseq_str(RSEQ_SIG) "\n\t" \
	198	__rseq_str(label) ":\n\t" \
	199	teardown \
	200	"jmp %l[" __rseq_str(abort_label) "]\n\t" \
	201	".popsection\n\t"
	202
abac6819 MD	203	/*
	204	* Define a critical section teardown handler.
	205	*
	206	* @label:
	207	* Local label to the teardown handler.
	208	* @teardown:
	209	* Sequence of instructions to run on teardown.
	210	* @target_label:
	211	* C label to jump to at the end of the sequence.
	212	*/
559d824f	213	#define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label) \
784b0012 MD	214	".pushsection __rseq_failure, \"ax\"\n\t" \
	215	__rseq_str(label) ":\n\t" \
	216	teardown \
559d824f	217	"jmp %l[" __rseq_str(target_label) "]\n\t" \
784b0012 MD	218	".popsection\n\t"
784b0012 MD	219
abac6819 MD	220	/* Jump to local label @label when @cpu_id != @current_cpu_id. */
	221	#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \
	222	RSEQ_INJECT_ASM(2) \
	223	"cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
	224	"jnz " __rseq_str(label) "\n\t"
784b0012	225
809f5ee3	226	/* Per-cpu-id indexing. */
784b0012	227
abf9e855	228	#define RSEQ_TEMPLATE_INDEX_CPU_ID
809f5ee3	229	#define RSEQ_TEMPLATE_MO_RELAXED
44ec21eb	230	#include "rseq/arch/x86/bits.h"
809f5ee3	231	#undef RSEQ_TEMPLATE_MO_RELAXED
784b0012	232
809f5ee3	233	#define RSEQ_TEMPLATE_MO_RELEASE
44ec21eb	234	#include "rseq/arch/x86/bits.h"
809f5ee3	235	#undef RSEQ_TEMPLATE_MO_RELEASE
abf9e855	236	#undef RSEQ_TEMPLATE_INDEX_CPU_ID
784b0012	237
809f5ee3	238	/* Per-mm-cid indexing. */
784b0012	239
abf9e855	240	#define RSEQ_TEMPLATE_INDEX_MM_CID
809f5ee3	241	#define RSEQ_TEMPLATE_MO_RELAXED
44ec21eb	242	#include "rseq/arch/x86/bits.h"
809f5ee3	243	#undef RSEQ_TEMPLATE_MO_RELAXED
784b0012	244
809f5ee3	245	#define RSEQ_TEMPLATE_MO_RELEASE
44ec21eb	246	#include "rseq/arch/x86/bits.h"
809f5ee3	247	#undef RSEQ_TEMPLATE_MO_RELEASE
abf9e855	248	#undef RSEQ_TEMPLATE_INDEX_MM_CID
de28c254	249
abf9e855	250	/* APIs which are not indexed. */
de28c254	251
abf9e855	252	#define RSEQ_TEMPLATE_INDEX_NONE
809f5ee3	253	#define RSEQ_TEMPLATE_MO_RELAXED
44ec21eb	254	#include "rseq/arch/x86/bits.h"
809f5ee3	255	#undef RSEQ_TEMPLATE_MO_RELAXED
abf9e855	256	#undef RSEQ_TEMPLATE_INDEX_NONE