[librseq.git] / include / rseq / rseq-x86.h

/* SPDX-License-Identifier: MIT */
/* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */

/*
 * rseq-x86.h
 */

#ifndef RSEQ_H
#error "Never use <rseq-x86.h> directly; include <rseq.h> instead."
#endif

#include <stdint.h>

/*
 * RSEQ_SIG is used with the following reserved undefined instructions, which
 * trap in user-space:
 *
 * x86-32:    0f b9 3d 53 30 05 53      ud1    0x53053053,%edi
 * x86-64:    0f b9 3d 53 30 05 53      ud1    0x53053053(%rip),%edi
 */
#define RSEQ_SIG	0x53053053

/*
 * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
 * operands, we cannot use "m" input operands, and rather pass the __rseq_abi
 * address through a "r" input operand.
 */

/* Offset of cpu_id, rseq_cs, and mm_cid fields in struct rseq. */
#define RSEQ_CPU_ID_OFFSET	4
#define RSEQ_CS_OFFSET		8
#define RSEQ_MM_CID_OFFSET	24

#ifdef __x86_64__

#define RSEQ_ASM_TP_SEGMENT	%%fs

#define rseq_smp_mb()	\
	__asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
#define rseq_smp_rmb()	rseq_barrier()
#define rseq_smp_wmb()	rseq_barrier()

#define rseq_smp_load_acquire(p)					\
__extension__ ({							\
	rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p));	\
	rseq_barrier();							\
	____p1;								\
})

#define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_rmb()

#define rseq_smp_store_release(p, v)					\
do {									\
	rseq_barrier();							\
	RSEQ_WRITE_ONCE(*(p), v);					\
} while (0)

#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,			\
				start_ip, post_commit_offset, abort_ip)	\
		".pushsection __rseq_cs, \"aw\"\n\t"			\
		".balign 32\n\t"					\
		__rseq_str(label) ":\n\t"				\
		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
		".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
		".popsection\n\t"					\
		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
		".quad " __rseq_str(label) "b\n\t"			\
		".popsection\n\t"


#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,		\
				(post_commit_ip - start_ip), abort_ip)

/*
 * Exit points of a rseq critical section consist of all instructions outside
 * of the critical section where a critical section can either branch to or
 * reach through the normal course of its execution. The abort IP and the
 * post-commit IP are already part of the __rseq_cs section and should not be
 * explicitly defined as additional exit points. Knowing all exit points is
 * useful to assist debuggers stepping over the critical section.
 */
#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
		".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
		".popsection\n\t"

#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)		\
		RSEQ_INJECT_ASM(1)					\
		"leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t"	\
		"movq %%rax, " __rseq_str(rseq_cs) "\n\t"		\
		__rseq_str(label) ":\n\t"

#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label)		\
		RSEQ_INJECT_ASM(2)					\
		"cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
		"jnz " __rseq_str(label) "\n\t"

#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)		\
		".pushsection __rseq_failure, \"ax\"\n\t"		\
		/* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
		".byte 0x0f, 0xb9, 0x3d\n\t"				\
		".long " __rseq_str(RSEQ_SIG) "\n\t"			\
		__rseq_str(label) ":\n\t"				\
		teardown						\
		"jmp %l[" __rseq_str(abort_label) "]\n\t"		\
		".popsection\n\t"

#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)		\
		".pushsection __rseq_failure, \"ax\"\n\t"		\
		__rseq_str(label) ":\n\t"				\
		teardown						\
		"jmp %l[" __rseq_str(cmpfail_label) "]\n\t"		\
		".popsection\n\t"

#elif defined(__i386__)

#define RSEQ_ASM_TP_SEGMENT	%%gs

#define rseq_smp_mb()	\
	__asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
#define rseq_smp_rmb()	\
	__asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
#define rseq_smp_wmb()	\
	__asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")

#define rseq_smp_load_acquire(p)					\
__extension__ ({							\
	rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p));	\
	rseq_smp_mb();							\
	____p1;								\
})

#define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_rmb()

#define rseq_smp_store_release(p, v)					\
do {									\
	rseq_smp_mb();							\
	RSEQ_WRITE_ONCE(*p, v);						\
} while (0)

/*
 * Use eax as scratch register and take memory operands as input to
 * lessen register pressure. Especially needed when compiling in O0.
 */
#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,			\
				start_ip, post_commit_offset, abort_ip)	\
		".pushsection __rseq_cs, \"aw\"\n\t"			\
		".balign 32\n\t"					\
		__rseq_str(label) ":\n\t"				\
		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
		".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
		".popsection\n\t"					\
		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
		".long " __rseq_str(label) "b, 0x0\n\t"			\
		".popsection\n\t"

#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,		\
				(post_commit_ip - start_ip), abort_ip)

/*
 * Exit points of a rseq critical section consist of all instructions outside
 * of the critical section where a critical section can either branch to or
 * reach through the normal course of its execution. The abort IP and the
 * post-commit IP are already part of the __rseq_cs section and should not be
 * explicitly defined as additional exit points. Knowing all exit points is
 * useful to assist debuggers stepping over the critical section.
 */
#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
		".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
		".popsection\n\t"

#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)		\
		RSEQ_INJECT_ASM(1)					\
		"movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t"	\
		__rseq_str(label) ":\n\t"

#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label)		\
		RSEQ_INJECT_ASM(2)					\
		"cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
		"jnz " __rseq_str(label) "\n\t"

#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)		\
		".pushsection __rseq_failure, \"ax\"\n\t"		\
		/* Disassembler-friendly signature: ud1 <sig>,%edi. */	\
		".byte 0x0f, 0xb9, 0x3d\n\t"				\
		".long " __rseq_str(RSEQ_SIG) "\n\t"			\
		__rseq_str(label) ":\n\t"				\
		teardown						\
		"jmp %l[" __rseq_str(abort_label) "]\n\t"		\
		".popsection\n\t"

#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)		\
		".pushsection __rseq_failure, \"ax\"\n\t"		\
		__rseq_str(label) ":\n\t"				\
		teardown						\
		"jmp %l[" __rseq_str(cmpfail_label) "]\n\t"		\
		".popsection\n\t"

#endif

/* Per-cpu-id indexing. */

#define RSEQ_TEMPLATE_CPU_ID
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq-x86-bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED

#define RSEQ_TEMPLATE_MO_RELEASE
#include "rseq-x86-bits.h"
#undef RSEQ_TEMPLATE_MO_RELEASE
#undef RSEQ_TEMPLATE_CPU_ID

/* Per-mm-cid indexing. */

#define RSEQ_TEMPLATE_MM_CID
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq-x86-bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED

#define RSEQ_TEMPLATE_MO_RELEASE
#include "rseq-x86-bits.h"
#undef RSEQ_TEMPLATE_MO_RELEASE
#undef RSEQ_TEMPLATE_MM_CID

/* APIs which are not based on cpu ids. */

#define RSEQ_TEMPLATE_CPU_ID_NONE
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq-x86-bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED
#undef RSEQ_TEMPLATE_CPU_ID_NONE
Commit	Line	Data
90702366	1	/* SPDX-License-Identifier: MIT */
f2d7b530 MJ	2	/* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
f2d7b530 MJ	3
784b0012 MD	4	/*
784b0012 MD	5	* rseq-x86.h
784b0012 MD	6	*/
784b0012 MD	7
809f5ee3 MD	8	#ifndef RSEQ_H
	9	#error "Never use <rseq-x86.h> directly; include <rseq.h> instead."
	10	#endif
	11
784b0012 MD	12	#include <stdint.h>
784b0012 MD	13
a5fe9e95 MD	14	/*
	15	* RSEQ_SIG is used with the following reserved undefined instructions, which
	16	* trap in user-space:
	17	*
	18	* x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi
	19	* x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi
	20	*/
784b0012 MD	21	#define RSEQ_SIG 0x53053053
784b0012 MD	22
809f5ee3 MD	23	/*
	24	* Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
	25	* operands, we cannot use "m" input operands, and rather pass the __rseq_abi
	26	* address through a "r" input operand.
	27	*/
	28
	29	/* Offset of cpu_id, rseq_cs, and mm_cid fields in struct rseq. */
05ebd3f9 MD	30	#define RSEQ_CPU_ID_OFFSET 4
05ebd3f9 MD	31	#define RSEQ_CS_OFFSET 8
809f5ee3	32	#define RSEQ_MM_CID_OFFSET 24
05ebd3f9	33
784b0012 MD	34	#ifdef __x86_64__
784b0012 MD	35
7dc4bc29 MD	36	#define RSEQ_ASM_TP_SEGMENT %%fs
7dc4bc29 MD	37
784b0012 MD	38	#define rseq_smp_mb() \
	39	__asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
	40	#define rseq_smp_rmb() rseq_barrier()
	41	#define rseq_smp_wmb() rseq_barrier()
	42
	43	#define rseq_smp_load_acquire(p) \
	44	__extension__ ({ \
3664a718	45	rseq_unqual_scalar_typeof((p)) ____p1 = RSEQ_READ_ONCE((p)); \
784b0012 MD	46	rseq_barrier(); \
	47	____p1; \
	48	})
	49
	50	#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
	51
	52	#define rseq_smp_store_release(p, v) \
	53	do { \
	54	rseq_barrier(); \
809f5ee3	55	RSEQ_WRITE_ONCE(*(p), v); \
784b0012 MD	56	} while (0)
784b0012 MD	57
784b0012 MD	58	#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
784b0012 MD	59	start_ip, post_commit_offset, abort_ip) \
dd01d0fb	60	".pushsection __rseq_cs, \"aw\"\n\t" \
784b0012 MD	61	".balign 32\n\t" \
	62	__rseq_str(label) ":\n\t" \
	63	".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
	64	".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
dd01d0fb MD	65	".popsection\n\t" \
	66	".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
	67	".quad " __rseq_str(label) "b\n\t" \
784b0012 MD	68	".popsection\n\t"
784b0012 MD	69
dd01d0fb	70
784b0012 MD	71	#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
	72	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
	73	(post_commit_ip - start_ip), abort_ip)
	74
90d9876e MD	75	/*
90d9876e MD	76	* Exit points of a rseq critical section consist of all instructions outside
fd622cad MD	77	* of the critical section where a critical section can either branch to or
	78	* reach through the normal course of its execution. The abort IP and the
	79	* post-commit IP are already part of the __rseq_cs section and should not be
	80	* explicitly defined as additional exit points. Knowing all exit points is
	81	* useful to assist debuggers stepping over the critical section.
90d9876e MD	82	*/
	83	#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
	84	".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
	85	".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
	86	".popsection\n\t"
	87
784b0012 MD	88	#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
	89	RSEQ_INJECT_ASM(1) \
	90	"leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \
05ebd3f9	91	"movq %%rax, " __rseq_str(rseq_cs) "\n\t" \
784b0012 MD	92	__rseq_str(label) ":\n\t"
784b0012 MD	93
769ec9a5	94	#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \
784b0012	95	RSEQ_INJECT_ASM(2) \
05ebd3f9	96	"cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
784b0012 MD	97	"jnz " __rseq_str(label) "\n\t"
	98
	99	#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
	100	".pushsection __rseq_failure, \"ax\"\n\t" \
a5fe9e95 MD	101	/* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
a5fe9e95 MD	102	".byte 0x0f, 0xb9, 0x3d\n\t" \
784b0012 MD	103	".long " __rseq_str(RSEQ_SIG) "\n\t" \
	104	__rseq_str(label) ":\n\t" \
	105	teardown \
	106	"jmp %l[" __rseq_str(abort_label) "]\n\t" \
	107	".popsection\n\t"
	108
	109	#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
	110	".pushsection __rseq_failure, \"ax\"\n\t" \
	111	__rseq_str(label) ":\n\t" \
	112	teardown \
	113	"jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
	114	".popsection\n\t"
	115
f1c6b55b	116	#elif defined(__i386__)
784b0012	117
53831176 MD	118	#define RSEQ_ASM_TP_SEGMENT %%gs
53831176 MD	119
784b0012 MD	120	#define rseq_smp_mb() \
	121	__asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
	122	#define rseq_smp_rmb() \
	123	__asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
	124	#define rseq_smp_wmb() \
	125	__asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
	126
	127	#define rseq_smp_load_acquire(p) \
	128	__extension__ ({ \
3664a718	129	rseq_unqual_scalar_typeof((p)) ____p1 = RSEQ_READ_ONCE((p)); \
784b0012 MD	130	rseq_smp_mb(); \
	131	____p1; \
	132	})
	133
	134	#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
	135
	136	#define rseq_smp_store_release(p, v) \
	137	do { \
	138	rseq_smp_mb(); \
809f5ee3	139	RSEQ_WRITE_ONCE(*p, v); \
784b0012 MD	140	} while (0)
784b0012 MD	141
784b0012 MD	142	/*
	143	* Use eax as scratch register and take memory operands as input to
	144	* lessen register pressure. Especially needed when compiling in O0.
	145	*/
	146	#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
	147	start_ip, post_commit_offset, abort_ip) \
dd01d0fb	148	".pushsection __rseq_cs, \"aw\"\n\t" \
784b0012 MD	149	".balign 32\n\t" \
	150	__rseq_str(label) ":\n\t" \
	151	".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
	152	".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
dd01d0fb MD	153	".popsection\n\t" \
	154	".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
	155	".long " __rseq_str(label) "b, 0x0\n\t" \
784b0012 MD	156	".popsection\n\t"
	157
	158	#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
	159	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
	160	(post_commit_ip - start_ip), abort_ip)
	161
90d9876e MD	162	/*
90d9876e MD	163	* Exit points of a rseq critical section consist of all instructions outside
fd622cad MD	164	* of the critical section where a critical section can either branch to or
	165	* reach through the normal course of its execution. The abort IP and the
	166	* post-commit IP are already part of the __rseq_cs section and should not be
	167	* explicitly defined as additional exit points. Knowing all exit points is
	168	* useful to assist debuggers stepping over the critical section.
90d9876e MD	169	*/
	170	#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
	171	".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
	172	".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
	173	".popsection\n\t"
	174
784b0012 MD	175	#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
784b0012 MD	176	RSEQ_INJECT_ASM(1) \
05ebd3f9	177	"movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \
784b0012 MD	178	__rseq_str(label) ":\n\t"
784b0012 MD	179
769ec9a5	180	#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \
784b0012	181	RSEQ_INJECT_ASM(2) \
05ebd3f9	182	"cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
784b0012 MD	183	"jnz " __rseq_str(label) "\n\t"
	184
	185	#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
	186	".pushsection __rseq_failure, \"ax\"\n\t" \
a5fe9e95 MD	187	/* Disassembler-friendly signature: ud1 <sig>,%edi. */ \
a5fe9e95 MD	188	".byte 0x0f, 0xb9, 0x3d\n\t" \
784b0012 MD	189	".long " __rseq_str(RSEQ_SIG) "\n\t" \
	190	__rseq_str(label) ":\n\t" \
	191	teardown \
	192	"jmp %l[" __rseq_str(abort_label) "]\n\t" \
	193	".popsection\n\t"
	194
	195	#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
	196	".pushsection __rseq_failure, \"ax\"\n\t" \
	197	__rseq_str(label) ":\n\t" \
	198	teardown \
	199	"jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
	200	".popsection\n\t"
	201
784b0012	202	#endif
784b0012	203
809f5ee3	204	/* Per-cpu-id indexing. */
784b0012	205
809f5ee3 MD	206	#define RSEQ_TEMPLATE_CPU_ID
	207	#define RSEQ_TEMPLATE_MO_RELAXED
	208	#include "rseq-x86-bits.h"
	209	#undef RSEQ_TEMPLATE_MO_RELAXED
784b0012	210
809f5ee3 MD	211	#define RSEQ_TEMPLATE_MO_RELEASE
	212	#include "rseq-x86-bits.h"
	213	#undef RSEQ_TEMPLATE_MO_RELEASE
	214	#undef RSEQ_TEMPLATE_CPU_ID
784b0012	215
809f5ee3	216	/* Per-mm-cid indexing. */
784b0012	217
809f5ee3 MD	218	#define RSEQ_TEMPLATE_MM_CID
	219	#define RSEQ_TEMPLATE_MO_RELAXED
	220	#include "rseq-x86-bits.h"
	221	#undef RSEQ_TEMPLATE_MO_RELAXED
784b0012	222
809f5ee3 MD	223	#define RSEQ_TEMPLATE_MO_RELEASE
	224	#include "rseq-x86-bits.h"
	225	#undef RSEQ_TEMPLATE_MO_RELEASE
	226	#undef RSEQ_TEMPLATE_MM_CID
de28c254	227
809f5ee3	228	/* APIs which are not based on cpu ids. */
de28c254	229
809f5ee3 MD	230	#define RSEQ_TEMPLATE_CPU_ID_NONE
	231	#define RSEQ_TEMPLATE_MO_RELAXED
	232	#include "rseq-x86-bits.h"
	233	#undef RSEQ_TEMPLATE_MO_RELAXED
	234	#undef RSEQ_TEMPLATE_CPU_ID_NONE