[librseq.git] / include / rseq / rseq-x86.h

/* SPDX-License-Identifier: MIT */
/* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */

/*
 * rseq-x86.h
 */

#ifndef RSEQ_H
#error "Never use <rseq-x86.h> directly; include <rseq.h> instead."
#endif

#include <stdint.h>

/*
 * RSEQ_SIG is used with the following reserved undefined instructions, which
 * trap in user-space:
 *
 * x86-32:    0f b9 3d 53 30 05 53      ud1    0x53053053,%edi
 * x86-64:    0f b9 3d 53 30 05 53      ud1    0x53053053(%rip),%edi
 */
#define RSEQ_SIG	0x53053053

/*
 * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
 * operands, we cannot use "m" input operands, and rather pass the __rseq_abi
 * address through a "r" input operand.
 */

/* Offset of cpu_id, rseq_cs, and mm_cid fields in struct rseq. */
#define RSEQ_ASM_CPU_ID_OFFSET		4
#define RSEQ_ASM_CS_OFFSET		8
#define RSEQ_ASM_MM_CID_OFFSET		24

#define rseq_smp_mb()	\
	__asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
#define rseq_smp_rmb()	rseq_barrier()
#define rseq_smp_wmb()	rseq_barrier()

#define rseq_smp_load_acquire(p)					\
__extension__ ({							\
	rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p));	\
	rseq_barrier();							\
	____p1;								\
})

#define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_rmb()

#define rseq_smp_store_release(p, v)					\
do {									\
	rseq_barrier();							\
	RSEQ_WRITE_ONCE(*(p), v);					\
} while (0)

#ifdef __x86_64__

#define RSEQ_ASM_TP_SEGMENT	%%fs

#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,			\
				start_ip, post_commit_offset, abort_ip)	\
		".pushsection __rseq_cs, \"aw\"\n\t"			\
		".balign 32\n\t"					\
		__rseq_str(label) ":\n\t"				\
		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
		".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
		".popsection\n\t"					\
		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
		".quad " __rseq_str(label) "b\n\t"			\
		".popsection\n\t"


#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,		\
				(post_commit_ip - start_ip), abort_ip)

/*
 * Exit points of a rseq critical section consist of all instructions outside
 * of the critical section where a critical section can either branch to or
 * reach through the normal course of its execution. The abort IP and the
 * post-commit IP are already part of the __rseq_cs section and should not be
 * explicitly defined as additional exit points. Knowing all exit points is
 * useful to assist debuggers stepping over the critical section.
 */
#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
		".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
		".popsection\n\t"

#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)		\
		RSEQ_INJECT_ASM(1)					\
		"leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t"	\
		"movq %%rax, " __rseq_str(rseq_cs) "\n\t"		\
		__rseq_str(label) ":\n\t"

#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label)		\
		RSEQ_INJECT_ASM(2)					\
		"cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
		"jnz " __rseq_str(label) "\n\t"

#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)		\
		".pushsection __rseq_failure, \"ax\"\n\t"		\
		/* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
		".byte 0x0f, 0xb9, 0x3d\n\t"				\
		".long " __rseq_str(RSEQ_SIG) "\n\t"			\
		__rseq_str(label) ":\n\t"				\
		teardown						\
		"jmp %l[" __rseq_str(abort_label) "]\n\t"		\
		".popsection\n\t"

#define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label)		\
		".pushsection __rseq_failure, \"ax\"\n\t"		\
		__rseq_str(label) ":\n\t"				\
		teardown						\
		"jmp %l[" __rseq_str(target_label) "]\n\t"		\
		".popsection\n\t"

#elif defined(__i386__)

#define RSEQ_ASM_TP_SEGMENT	%%gs

/*
 * Use eax as scratch register and take memory operands as input to
 * lessen register pressure. Especially needed when compiling in O0.
 */
#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,			\
				start_ip, post_commit_offset, abort_ip)	\
		".pushsection __rseq_cs, \"aw\"\n\t"			\
		".balign 32\n\t"					\
		__rseq_str(label) ":\n\t"				\
		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
		".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
		".popsection\n\t"					\
		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
		".long " __rseq_str(label) "b, 0x0\n\t"			\
		".popsection\n\t"

#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,		\
				(post_commit_ip - start_ip), abort_ip)

/*
 * Exit points of a rseq critical section consist of all instructions outside
 * of the critical section where a critical section can either branch to or
 * reach through the normal course of its execution. The abort IP and the
 * post-commit IP are already part of the __rseq_cs section and should not be
 * explicitly defined as additional exit points. Knowing all exit points is
 * useful to assist debuggers stepping over the critical section.
 */
#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
		".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
		".popsection\n\t"

#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)		\
		RSEQ_INJECT_ASM(1)					\
		"movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t"	\
		__rseq_str(label) ":\n\t"

#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label)		\
		RSEQ_INJECT_ASM(2)					\
		"cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
		"jnz " __rseq_str(label) "\n\t"

#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)		\
		".pushsection __rseq_failure, \"ax\"\n\t"		\
		/* Disassembler-friendly signature: ud1 <sig>,%edi. */	\
		".byte 0x0f, 0xb9, 0x3d\n\t"				\
		".long " __rseq_str(RSEQ_SIG) "\n\t"			\
		__rseq_str(label) ":\n\t"				\
		teardown						\
		"jmp %l[" __rseq_str(abort_label) "]\n\t"		\
		".popsection\n\t"

#define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label)		\
		".pushsection __rseq_failure, \"ax\"\n\t"		\
		__rseq_str(label) ":\n\t"				\
		teardown						\
		"jmp %l[" __rseq_str(target_label) "]\n\t"		\
		".popsection\n\t"

#endif

/* Per-cpu-id indexing. */

#define RSEQ_TEMPLATE_CPU_ID
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq-x86-bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED

#define RSEQ_TEMPLATE_MO_RELEASE
#include "rseq-x86-bits.h"
#undef RSEQ_TEMPLATE_MO_RELEASE
#undef RSEQ_TEMPLATE_CPU_ID

/* Per-mm-cid indexing. */

#define RSEQ_TEMPLATE_MM_CID
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq-x86-bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED

#define RSEQ_TEMPLATE_MO_RELEASE
#include "rseq-x86-bits.h"
#undef RSEQ_TEMPLATE_MO_RELEASE
#undef RSEQ_TEMPLATE_MM_CID

/* APIs which are not based on cpu ids. */

#define RSEQ_TEMPLATE_CPU_ID_NONE
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq-x86-bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED
#undef RSEQ_TEMPLATE_CPU_ID_NONE
Commit	Line	Data
90702366	1	/* SPDX-License-Identifier: MIT */
f2d7b530 MJ	2	/* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
f2d7b530 MJ	3
784b0012 MD	4	/*
784b0012 MD	5	* rseq-x86.h
784b0012 MD	6	*/
784b0012 MD	7
809f5ee3 MD	8	#ifndef RSEQ_H
	9	#error "Never use <rseq-x86.h> directly; include <rseq.h> instead."
	10	#endif
	11
784b0012 MD	12	#include <stdint.h>
784b0012 MD	13
a5fe9e95 MD	14	/*
	15	* RSEQ_SIG is used with the following reserved undefined instructions, which
	16	* trap in user-space:
	17	*
	18	* x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi
	19	* x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi
	20	*/
784b0012 MD	21	#define RSEQ_SIG 0x53053053
784b0012 MD	22
809f5ee3 MD	23	/*
	24	* Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
	25	* operands, we cannot use "m" input operands, and rather pass the __rseq_abi
	26	* address through a "r" input operand.
	27	*/
	28
	29	/* Offset of cpu_id, rseq_cs, and mm_cid fields in struct rseq. */
d9c11a15 MD	30	#define RSEQ_ASM_CPU_ID_OFFSET 4
	31	#define RSEQ_ASM_CS_OFFSET 8
	32	#define RSEQ_ASM_MM_CID_OFFSET 24
05ebd3f9	33
784b0012 MD	34	#define rseq_smp_mb() \
	35	__asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
	36	#define rseq_smp_rmb() rseq_barrier()
	37	#define rseq_smp_wmb() rseq_barrier()
	38
	39	#define rseq_smp_load_acquire(p) \
	40	__extension__ ({ \
3664a718	41	rseq_unqual_scalar_typeof((p)) ____p1 = RSEQ_READ_ONCE((p)); \
784b0012 MD	42	rseq_barrier(); \
	43	____p1; \
	44	})
	45
	46	#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
	47
	48	#define rseq_smp_store_release(p, v) \
	49	do { \
	50	rseq_barrier(); \
809f5ee3	51	RSEQ_WRITE_ONCE(*(p), v); \
784b0012 MD	52	} while (0)
784b0012 MD	53
94f43fc9 MD	54	#ifdef __x86_64__
	55
	56	#define RSEQ_ASM_TP_SEGMENT %%fs
	57
784b0012 MD	58	#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
784b0012 MD	59	start_ip, post_commit_offset, abort_ip) \
dd01d0fb	60	".pushsection __rseq_cs, \"aw\"\n\t" \
784b0012 MD	61	".balign 32\n\t" \
	62	__rseq_str(label) ":\n\t" \
	63	".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
	64	".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
dd01d0fb MD	65	".popsection\n\t" \
	66	".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
	67	".quad " __rseq_str(label) "b\n\t" \
784b0012 MD	68	".popsection\n\t"
784b0012 MD	69
dd01d0fb	70
784b0012 MD	71	#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
	72	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
	73	(post_commit_ip - start_ip), abort_ip)
	74
90d9876e MD	75	/*
90d9876e MD	76	* Exit points of a rseq critical section consist of all instructions outside
fd622cad MD	77	* of the critical section where a critical section can either branch to or
	78	* reach through the normal course of its execution. The abort IP and the
	79	* post-commit IP are already part of the __rseq_cs section and should not be
	80	* explicitly defined as additional exit points. Knowing all exit points is
	81	* useful to assist debuggers stepping over the critical section.
90d9876e MD	82	*/
	83	#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
	84	".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
	85	".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
	86	".popsection\n\t"
	87
784b0012 MD	88	#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
	89	RSEQ_INJECT_ASM(1) \
	90	"leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \
05ebd3f9	91	"movq %%rax, " __rseq_str(rseq_cs) "\n\t" \
784b0012 MD	92	__rseq_str(label) ":\n\t"
784b0012 MD	93
769ec9a5	94	#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \
784b0012	95	RSEQ_INJECT_ASM(2) \
05ebd3f9	96	"cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
784b0012 MD	97	"jnz " __rseq_str(label) "\n\t"
	98
	99	#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
	100	".pushsection __rseq_failure, \"ax\"\n\t" \
a5fe9e95 MD	101	/* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
a5fe9e95 MD	102	".byte 0x0f, 0xb9, 0x3d\n\t" \
784b0012 MD	103	".long " __rseq_str(RSEQ_SIG) "\n\t" \
	104	__rseq_str(label) ":\n\t" \
	105	teardown \
	106	"jmp %l[" __rseq_str(abort_label) "]\n\t" \
	107	".popsection\n\t"
	108
559d824f	109	#define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label) \
784b0012 MD	110	".pushsection __rseq_failure, \"ax\"\n\t" \
	111	__rseq_str(label) ":\n\t" \
	112	teardown \
559d824f	113	"jmp %l[" __rseq_str(target_label) "]\n\t" \
784b0012 MD	114	".popsection\n\t"
784b0012 MD	115
f1c6b55b	116	#elif defined(__i386__)
784b0012	117
53831176 MD	118	#define RSEQ_ASM_TP_SEGMENT %%gs
53831176 MD	119
784b0012 MD	120	/*
	121	* Use eax as scratch register and take memory operands as input to
	122	* lessen register pressure. Especially needed when compiling in O0.
	123	*/
	124	#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
	125	start_ip, post_commit_offset, abort_ip) \
dd01d0fb	126	".pushsection __rseq_cs, \"aw\"\n\t" \
784b0012 MD	127	".balign 32\n\t" \
	128	__rseq_str(label) ":\n\t" \
	129	".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
	130	".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
dd01d0fb MD	131	".popsection\n\t" \
	132	".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
	133	".long " __rseq_str(label) "b, 0x0\n\t" \
784b0012 MD	134	".popsection\n\t"
	135
	136	#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
	137	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
	138	(post_commit_ip - start_ip), abort_ip)
	139
90d9876e MD	140	/*
90d9876e MD	141	* Exit points of a rseq critical section consist of all instructions outside
fd622cad MD	142	* of the critical section where a critical section can either branch to or
	143	* reach through the normal course of its execution. The abort IP and the
	144	* post-commit IP are already part of the __rseq_cs section and should not be
	145	* explicitly defined as additional exit points. Knowing all exit points is
	146	* useful to assist debuggers stepping over the critical section.
90d9876e MD	147	*/
	148	#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
	149	".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
	150	".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
	151	".popsection\n\t"
	152
784b0012 MD	153	#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
784b0012 MD	154	RSEQ_INJECT_ASM(1) \
05ebd3f9	155	"movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \
784b0012 MD	156	__rseq_str(label) ":\n\t"
784b0012 MD	157
769ec9a5	158	#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \
784b0012	159	RSEQ_INJECT_ASM(2) \
05ebd3f9	160	"cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
784b0012 MD	161	"jnz " __rseq_str(label) "\n\t"
	162
	163	#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
	164	".pushsection __rseq_failure, \"ax\"\n\t" \
a5fe9e95 MD	165	/* Disassembler-friendly signature: ud1 <sig>,%edi. */ \
a5fe9e95 MD	166	".byte 0x0f, 0xb9, 0x3d\n\t" \
784b0012 MD	167	".long " __rseq_str(RSEQ_SIG) "\n\t" \
	168	__rseq_str(label) ":\n\t" \
	169	teardown \
	170	"jmp %l[" __rseq_str(abort_label) "]\n\t" \
	171	".popsection\n\t"
	172
559d824f	173	#define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label) \
784b0012 MD	174	".pushsection __rseq_failure, \"ax\"\n\t" \
	175	__rseq_str(label) ":\n\t" \
	176	teardown \
559d824f	177	"jmp %l[" __rseq_str(target_label) "]\n\t" \
784b0012 MD	178	".popsection\n\t"
784b0012 MD	179
784b0012	180	#endif
784b0012	181
809f5ee3	182	/* Per-cpu-id indexing. */
784b0012	183
809f5ee3 MD	184	#define RSEQ_TEMPLATE_CPU_ID
	185	#define RSEQ_TEMPLATE_MO_RELAXED
	186	#include "rseq-x86-bits.h"
	187	#undef RSEQ_TEMPLATE_MO_RELAXED
784b0012	188
809f5ee3 MD	189	#define RSEQ_TEMPLATE_MO_RELEASE
	190	#include "rseq-x86-bits.h"
	191	#undef RSEQ_TEMPLATE_MO_RELEASE
	192	#undef RSEQ_TEMPLATE_CPU_ID
784b0012	193
809f5ee3	194	/* Per-mm-cid indexing. */
784b0012	195
809f5ee3 MD	196	#define RSEQ_TEMPLATE_MM_CID
	197	#define RSEQ_TEMPLATE_MO_RELAXED
	198	#include "rseq-x86-bits.h"
	199	#undef RSEQ_TEMPLATE_MO_RELAXED
784b0012	200
809f5ee3 MD	201	#define RSEQ_TEMPLATE_MO_RELEASE
	202	#include "rseq-x86-bits.h"
	203	#undef RSEQ_TEMPLATE_MO_RELEASE
	204	#undef RSEQ_TEMPLATE_MM_CID
de28c254	205
809f5ee3	206	/* APIs which are not based on cpu ids. */
de28c254	207
809f5ee3 MD	208	#define RSEQ_TEMPLATE_CPU_ID_NONE
	209	#define RSEQ_TEMPLATE_MO_RELAXED
	210	#include "rseq-x86-bits.h"
	211	#undef RSEQ_TEMPLATE_MO_RELAXED
	212	#undef RSEQ_TEMPLATE_CPU_ID_NONE