[librseq.git] / include / rseq / rseq-x86.h

/* SPDX-License-Identifier: MIT */
/* SPDX-FileCopyrightText: 2016-2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */

/*
 * rseq-x86.h
 */

#ifndef RSEQ_H
#error "Never use <rseq-x86.h> directly; include <rseq.h> instead."
#endif

#include <stdint.h>

/*
 * RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those
 * are not part of the public API.
 */

/*
 * RSEQ_SIG is used with the following reserved undefined instructions, which
 * trap in user-space:
 *
 * x86-32:    0f b9 3d 53 30 05 53      ud1    0x53053053,%edi
 * x86-64:    0f b9 3d 53 30 05 53      ud1    0x53053053(%rip),%edi
 */
#define RSEQ_SIG	0x53053053

/*
 * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
 * operands, we cannot use "m" input operands, and rather pass the __rseq_abi
 * address through a "r" input operand.
 */

/*
 * Offset of cpu_id, rseq_cs, and mm_cid fields in struct rseq. Those
 * are defined explicitly as macros to be used from assembly.
 */
#define RSEQ_ASM_CPU_ID_OFFSET		4
#define RSEQ_ASM_CS_OFFSET		8
#define RSEQ_ASM_MM_CID_OFFSET		24

/*
 * Refer to the Linux kernel memory model (LKMM) for documentation of
 * the memory barriers. Expect all x86 hardware to be x86-TSO (Total
 * Store Order).
 */

/* CPU memory barrier. */
#define rseq_smp_mb()	\
	__asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
/* CPU read memory barrier */
#define rseq_smp_rmb()	rseq_barrier()
/* CPU write memory barrier */
#define rseq_smp_wmb()	rseq_barrier()

/* Acquire: One-way permeable barrier. */
#define rseq_smp_load_acquire(p)					\
__extension__ ({							\
	rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p));	\
	rseq_barrier();							\
	____p1;								\
})

/* Acquire barrier after control dependency. */
#define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_rmb()

/* Release: One-way permeable barrier. */
#define rseq_smp_store_release(p, v)					\
do {									\
	rseq_barrier();							\
	RSEQ_WRITE_ONCE(*(p), v);					\
} while (0)

#ifdef __x86_64__

/* Segment selector for the thread pointer. */
#define RSEQ_ASM_TP_SEGMENT	%%fs

/* Only used in RSEQ_ASM_DEFINE_TABLE. */
#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,			\
				start_ip, post_commit_offset, abort_ip)	\
		".pushsection __rseq_cs, \"aw\"\n\t"			\
		".balign 32\n\t"					\
		__rseq_str(label) ":\n\t"				\
		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
		".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
		".popsection\n\t"					\
		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
		".quad " __rseq_str(label) "b\n\t"			\
		".popsection\n\t"

/*
 * Define the @exit_ip pointer as an exit point for the sequence of consecutive
 * assembly instructions at @start_ip.
 *
 *  @start_ip:
 *    Pointer to the first instruction of the sequence of consecutive assembly
 *    instructions.
 *  @exit_ip:
 *    Pointer to an exit point instruction.
 *
 * Exit points of a rseq critical section consist of all instructions outside
 * of the critical section where a critical section can either branch to or
 * reach through the normal course of its execution. The abort IP and the
 * post-commit IP are already part of the __rseq_cs section and should not be
 * explicitly defined as additional exit points. Knowing all exit points is
 * useful to assist debuggers stepping over the critical section.
 */
#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
		".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
		".popsection\n\t"

/*
 * Store the address of the critical section descriptor structure at
 * @cs_label into the @rseq_cs pointer and emit the label @label, which
 * is the beginning of the sequence of consecutive assembly instructions.
 *
 *  @label:
 *    Local label to the beginning of the sequence of consecutive assembly
 *    instructions.
 *  @cs_label:
 *    Source local label to the critical section descriptor structure.
 *  @rseq_cs:
 *    Destination pointer where to store the address of the critical
 *    section descriptor structure.
 */
#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)		\
		RSEQ_INJECT_ASM(1)					\
		"leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t"	\
		"movq %%rax, " __rseq_str(rseq_cs) "\n\t"		\
		__rseq_str(label) ":\n\t"

#elif defined(__i386__)

/* Segment selector for the thread pointer. */
#define RSEQ_ASM_TP_SEGMENT	%%gs

/* Only used in RSEQ_ASM_DEFINE_TABLE. */
#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,			\
				start_ip, post_commit_offset, abort_ip)	\
		".pushsection __rseq_cs, \"aw\"\n\t"			\
		".balign 32\n\t"					\
		__rseq_str(label) ":\n\t"				\
		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
		".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
		".popsection\n\t"					\
		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
		".long " __rseq_str(label) "b, 0x0\n\t"			\
		".popsection\n\t"

/*
 * Define the @exit_ip pointer as an exit point for the sequence of consecutive
 * assembly instructions at @start_ip.
 *
 *  @start_ip:
 *    Pointer to the first instruction of the sequence of consecutive assembly
 *    instructions.
 *  @exit_ip:
 *    Pointer to an exit point instruction.
 *
 * Exit points of a rseq critical section consist of all instructions outside
 * of the critical section where a critical section can either branch to or
 * reach through the normal course of its execution. The abort IP and the
 * post-commit IP are already part of the __rseq_cs section and should not be
 * explicitly defined as additional exit points. Knowing all exit points is
 * useful to assist debuggers stepping over the critical section.
 */
#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
		".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
		".popsection\n\t"

/*
 * Store the address of the critical section descriptor structure at
 * @cs_label into the @rseq_cs pointer and emit the label @label, which
 * is the beginning of the sequence of consecutive assembly instructions.
 *
 *  @label:
 *    Local label to the beginning of the sequence of consecutive assembly
 *    instructions.
 *  @cs_label:
 *    Source local label to the critical section descriptor structure.
 *  @rseq_cs:
 *    Destination pointer where to store the address of the critical
 *    section descriptor structure.
 */
#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)		\
		RSEQ_INJECT_ASM(1)					\
		"movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t"	\
		__rseq_str(label) ":\n\t"

#endif

/*
 * Define an rseq critical section structure of version 0 with no flags.
 *
 *  @label:
 *    Local label for the beginning of the critical section descriptor
 *    structure.
 *  @start_ip:
 *    Pointer to the first instruction of the sequence of consecutive assembly
 *    instructions.
 *  @post_commit_ip:
 *    Pointer to the instruction after the last instruction of the sequence of
 *    consecutive assembly instructions.
 *  @abort_ip:
 *    Pointer to the instruction where to move the execution flow in case of
 *    abort of the sequence of consecutive assembly instructions.
 */
#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,		\
				(post_commit_ip - start_ip), abort_ip)

/*
 * Define a critical section abort handler.
 *
 *  @label:
 *    Local label to the abort handler.
 *  @teardown:
 *    Sequence of instructions to run on abort.
 *  @abort_label:
 *    C label to jump to at the end of the sequence.
 */
#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)		\
		".pushsection __rseq_failure, \"ax\"\n\t"		\
		/*							\
		 * Disassembler-friendly signature:			\
		 *   x86-32: ud1 <sig>,%edi				\
		 *   x86-64: ud1 <sig>(%rip),%edi			\
		 */							\
		".byte 0x0f, 0xb9, 0x3d\n\t"				\
		".long " __rseq_str(RSEQ_SIG) "\n\t"			\
		__rseq_str(label) ":\n\t"				\
		teardown						\
		"jmp %l[" __rseq_str(abort_label) "]\n\t"		\
		".popsection\n\t"

/*
 * Define a critical section teardown handler.
 *
 *  @label:
 *    Local label to the teardown handler.
 *  @teardown:
 *    Sequence of instructions to run on teardown.
 *  @target_label:
 *    C label to jump to at the end of the sequence.
 */
#define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label)		\
		".pushsection __rseq_failure, \"ax\"\n\t"		\
		__rseq_str(label) ":\n\t"				\
		teardown						\
		"jmp %l[" __rseq_str(target_label) "]\n\t"		\
		".popsection\n\t"

/* Jump to local label @label when @cpu_id != @current_cpu_id. */
#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label)		\
		RSEQ_INJECT_ASM(2)					\
		"cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
		"jnz " __rseq_str(label) "\n\t"

/* Per-cpu-id indexing. */

#define RSEQ_TEMPLATE_INDEX_CPU_ID
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq-x86-bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED

#define RSEQ_TEMPLATE_MO_RELEASE
#include "rseq-x86-bits.h"
#undef RSEQ_TEMPLATE_MO_RELEASE
#undef RSEQ_TEMPLATE_INDEX_CPU_ID

/* Per-mm-cid indexing. */

#define RSEQ_TEMPLATE_INDEX_MM_CID
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq-x86-bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED

#define RSEQ_TEMPLATE_MO_RELEASE
#include "rseq-x86-bits.h"
#undef RSEQ_TEMPLATE_MO_RELEASE
#undef RSEQ_TEMPLATE_INDEX_MM_CID

/* APIs which are not indexed. */

#define RSEQ_TEMPLATE_INDEX_NONE
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq-x86-bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED
#undef RSEQ_TEMPLATE_INDEX_NONE
Commit	Line	Data
90702366	1	/* SPDX-License-Identifier: MIT */
abac6819	2	/* SPDX-FileCopyrightText: 2016-2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
f2d7b530	3
784b0012 MD	4	/*
784b0012 MD	5	* rseq-x86.h
784b0012 MD	6	*/
784b0012 MD	7
809f5ee3 MD	8	#ifndef RSEQ_H
	9	#error "Never use <rseq-x86.h> directly; include <rseq.h> instead."
	10	#endif
	11
784b0012 MD	12	#include <stdint.h>
784b0012 MD	13
abac6819 MD	14	/*
	15	* RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those
	16	* are not part of the public API.
	17	*/
	18
a5fe9e95 MD	19	/*
	20	* RSEQ_SIG is used with the following reserved undefined instructions, which
	21	* trap in user-space:
	22	*
	23	* x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi
	24	* x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi
	25	*/
784b0012 MD	26	#define RSEQ_SIG 0x53053053
784b0012 MD	27
809f5ee3 MD	28	/*
	29	* Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
	30	* operands, we cannot use "m" input operands, and rather pass the __rseq_abi
	31	* address through a "r" input operand.
	32	*/
	33
abac6819 MD	34	/*
	35	* Offset of cpu_id, rseq_cs, and mm_cid fields in struct rseq. Those
	36	* are defined explicitly as macros to be used from assembly.
	37	*/
d9c11a15 MD	38	#define RSEQ_ASM_CPU_ID_OFFSET 4
	39	#define RSEQ_ASM_CS_OFFSET 8
	40	#define RSEQ_ASM_MM_CID_OFFSET 24
05ebd3f9	41
abac6819 MD	42	/*
	43	* Refer to the Linux kernel memory model (LKMM) for documentation of
	44	* the memory barriers. Expect all x86 hardware to be x86-TSO (Total
	45	* Store Order).
	46	*/
	47
	48	/* CPU memory barrier. */
784b0012 MD	49	#define rseq_smp_mb() \
784b0012 MD	50	__asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
abac6819	51	/* CPU read memory barrier */
784b0012	52	#define rseq_smp_rmb() rseq_barrier()
abac6819	53	/* CPU write memory barrier */
784b0012 MD	54	#define rseq_smp_wmb() rseq_barrier()
784b0012 MD	55
abac6819	56	/* Acquire: One-way permeable barrier. */
784b0012 MD	57	#define rseq_smp_load_acquire(p) \
784b0012 MD	58	__extension__ ({ \
3664a718	59	rseq_unqual_scalar_typeof((p)) ____p1 = RSEQ_READ_ONCE((p)); \
784b0012 MD	60	rseq_barrier(); \
	61	____p1; \
	62	})
	63
abac6819	64	/* Acquire barrier after control dependency. */
784b0012 MD	65	#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
784b0012 MD	66
abac6819	67	/* Release: One-way permeable barrier. */
784b0012 MD	68	#define rseq_smp_store_release(p, v) \
	69	do { \
	70	rseq_barrier(); \
809f5ee3	71	RSEQ_WRITE_ONCE(*(p), v); \
784b0012 MD	72	} while (0)
784b0012 MD	73
94f43fc9 MD	74	#ifdef __x86_64__
94f43fc9 MD	75
abac6819	76	/* Segment selector for the thread pointer. */
94f43fc9 MD	77	#define RSEQ_ASM_TP_SEGMENT %%fs
94f43fc9 MD	78
abac6819	79	/* Only used in RSEQ_ASM_DEFINE_TABLE. */
784b0012 MD	80	#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
784b0012 MD	81	start_ip, post_commit_offset, abort_ip) \
dd01d0fb	82	".pushsection __rseq_cs, \"aw\"\n\t" \
784b0012 MD	83	".balign 32\n\t" \
	84	__rseq_str(label) ":\n\t" \
	85	".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
	86	".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
dd01d0fb MD	87	".popsection\n\t" \
	88	".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
	89	".quad " __rseq_str(label) "b\n\t" \
784b0012 MD	90	".popsection\n\t"
784b0012 MD	91
90d9876e	92	/*
abac6819 MD	93	* Define the @exit_ip pointer as an exit point for the sequence of consecutive
	94	* assembly instructions at @start_ip.
	95	*
	96	* @start_ip:
	97	* Pointer to the first instruction of the sequence of consecutive assembly
	98	* instructions.
	99	* @exit_ip:
	100	* Pointer to an exit point instruction.
	101	*
90d9876e	102	* Exit points of a rseq critical section consist of all instructions outside
fd622cad MD	103	* of the critical section where a critical section can either branch to or
	104	* reach through the normal course of its execution. The abort IP and the
	105	* post-commit IP are already part of the __rseq_cs section and should not be
	106	* explicitly defined as additional exit points. Knowing all exit points is
	107	* useful to assist debuggers stepping over the critical section.
90d9876e MD	108	*/
	109	#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
	110	".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
	111	".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
	112	".popsection\n\t"
	113
abac6819 MD	114	/*
	115	* Store the address of the critical section descriptor structure at
	116	* @cs_label into the @rseq_cs pointer and emit the label @label, which
	117	* is the beginning of the sequence of consecutive assembly instructions.
	118	*
	119	* @label:
	120	* Local label to the beginning of the sequence of consecutive assembly
	121	* instructions.
	122	* @cs_label:
	123	* Source local label to the critical section descriptor structure.
	124	* @rseq_cs:
	125	* Destination pointer where to store the address of the critical
	126	* section descriptor structure.
	127	*/
784b0012 MD	128	#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
	129	RSEQ_INJECT_ASM(1) \
	130	"leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \
05ebd3f9	131	"movq %%rax, " __rseq_str(rseq_cs) "\n\t" \
784b0012 MD	132	__rseq_str(label) ":\n\t"
784b0012 MD	133
f1c6b55b	134	#elif defined(__i386__)
784b0012	135
abac6819	136	/* Segment selector for the thread pointer. */
53831176 MD	137	#define RSEQ_ASM_TP_SEGMENT %%gs
53831176 MD	138
abac6819	139	/* Only used in RSEQ_ASM_DEFINE_TABLE. */
784b0012 MD	140	#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
784b0012 MD	141	start_ip, post_commit_offset, abort_ip) \
dd01d0fb	142	".pushsection __rseq_cs, \"aw\"\n\t" \
784b0012 MD	143	".balign 32\n\t" \
	144	__rseq_str(label) ":\n\t" \
	145	".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
	146	".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
dd01d0fb MD	147	".popsection\n\t" \
	148	".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
	149	".long " __rseq_str(label) "b, 0x0\n\t" \
784b0012 MD	150	".popsection\n\t"
784b0012 MD	151
90d9876e	152	/*
abac6819 MD	153	* Define the @exit_ip pointer as an exit point for the sequence of consecutive
	154	* assembly instructions at @start_ip.
	155	*
	156	* @start_ip:
	157	* Pointer to the first instruction of the sequence of consecutive assembly
	158	* instructions.
	159	* @exit_ip:
	160	* Pointer to an exit point instruction.
	161	*
90d9876e	162	* Exit points of a rseq critical section consist of all instructions outside
fd622cad MD	163	* of the critical section where a critical section can either branch to or
	164	* reach through the normal course of its execution. The abort IP and the
	165	* post-commit IP are already part of the __rseq_cs section and should not be
	166	* explicitly defined as additional exit points. Knowing all exit points is
	167	* useful to assist debuggers stepping over the critical section.
90d9876e MD	168	*/
	169	#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
	170	".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
	171	".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
	172	".popsection\n\t"
	173
abac6819 MD	174	/*
	175	* Store the address of the critical section descriptor structure at
	176	* @cs_label into the @rseq_cs pointer and emit the label @label, which
	177	* is the beginning of the sequence of consecutive assembly instructions.
	178	*
	179	* @label:
	180	* Local label to the beginning of the sequence of consecutive assembly
	181	* instructions.
	182	* @cs_label:
	183	* Source local label to the critical section descriptor structure.
	184	* @rseq_cs:
	185	* Destination pointer where to store the address of the critical
	186	* section descriptor structure.
	187	*/
784b0012 MD	188	#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
784b0012 MD	189	RSEQ_INJECT_ASM(1) \
05ebd3f9	190	"movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \
784b0012 MD	191	__rseq_str(label) ":\n\t"
784b0012 MD	192
abac6819	193	#endif
784b0012	194
abac6819 MD	195	/*
	196	* Define an rseq critical section structure of version 0 with no flags.
	197	*
	198	* @label:
	199	* Local label for the beginning of the critical section descriptor
	200	* structure.
	201	* @start_ip:
	202	* Pointer to the first instruction of the sequence of consecutive assembly
	203	* instructions.
	204	* @post_commit_ip:
	205	* Pointer to the instruction after the last instruction of the sequence of
	206	* consecutive assembly instructions.
	207	* @abort_ip:
	208	* Pointer to the instruction where to move the execution flow in case of
	209	* abort of the sequence of consecutive assembly instructions.
	210	*/
	211	#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
	212	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
	213	(post_commit_ip - start_ip), abort_ip)
	214
	215	/*
	216	* Define a critical section abort handler.
	217	*
	218	* @label:
	219	* Local label to the abort handler.
	220	* @teardown:
	221	* Sequence of instructions to run on abort.
	222	* @abort_label:
	223	* C label to jump to at the end of the sequence.
	224	*/
784b0012 MD	225	#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
784b0012 MD	226	".pushsection __rseq_failure, \"ax\"\n\t" \
abac6819 MD	227	/* \
	228	* Disassembler-friendly signature: \
	229	* x86-32: ud1 <sig>,%edi \
	230	* x86-64: ud1 <sig>(%rip),%edi \
	231	*/ \
a5fe9e95	232	".byte 0x0f, 0xb9, 0x3d\n\t" \
784b0012 MD	233	".long " __rseq_str(RSEQ_SIG) "\n\t" \
	234	__rseq_str(label) ":\n\t" \
	235	teardown \
	236	"jmp %l[" __rseq_str(abort_label) "]\n\t" \
	237	".popsection\n\t"
	238
abac6819 MD	239	/*
	240	* Define a critical section teardown handler.
	241	*
	242	* @label:
	243	* Local label to the teardown handler.
	244	* @teardown:
	245	* Sequence of instructions to run on teardown.
	246	* @target_label:
	247	* C label to jump to at the end of the sequence.
	248	*/
559d824f	249	#define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label) \
784b0012 MD	250	".pushsection __rseq_failure, \"ax\"\n\t" \
	251	__rseq_str(label) ":\n\t" \
	252	teardown \
559d824f	253	"jmp %l[" __rseq_str(target_label) "]\n\t" \
784b0012 MD	254	".popsection\n\t"
784b0012 MD	255
abac6819 MD	256	/* Jump to local label @label when @cpu_id != @current_cpu_id. */
	257	#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \
	258	RSEQ_INJECT_ASM(2) \
	259	"cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
	260	"jnz " __rseq_str(label) "\n\t"
784b0012	261
809f5ee3	262	/* Per-cpu-id indexing. */
784b0012	263
abf9e855	264	#define RSEQ_TEMPLATE_INDEX_CPU_ID
809f5ee3 MD	265	#define RSEQ_TEMPLATE_MO_RELAXED
	266	#include "rseq-x86-bits.h"
	267	#undef RSEQ_TEMPLATE_MO_RELAXED
784b0012	268
809f5ee3 MD	269	#define RSEQ_TEMPLATE_MO_RELEASE
	270	#include "rseq-x86-bits.h"
	271	#undef RSEQ_TEMPLATE_MO_RELEASE
abf9e855	272	#undef RSEQ_TEMPLATE_INDEX_CPU_ID
784b0012	273
809f5ee3	274	/* Per-mm-cid indexing. */
784b0012	275
abf9e855	276	#define RSEQ_TEMPLATE_INDEX_MM_CID
809f5ee3 MD	277	#define RSEQ_TEMPLATE_MO_RELAXED
	278	#include "rseq-x86-bits.h"
	279	#undef RSEQ_TEMPLATE_MO_RELAXED
784b0012	280
809f5ee3 MD	281	#define RSEQ_TEMPLATE_MO_RELEASE
	282	#include "rseq-x86-bits.h"
	283	#undef RSEQ_TEMPLATE_MO_RELEASE
abf9e855	284	#undef RSEQ_TEMPLATE_INDEX_MM_CID
de28c254	285
abf9e855	286	/* APIs which are not indexed. */
de28c254	287
abf9e855	288	#define RSEQ_TEMPLATE_INDEX_NONE
809f5ee3 MD	289	#define RSEQ_TEMPLATE_MO_RELAXED
	290	#include "rseq-x86-bits.h"
	291	#undef RSEQ_TEMPLATE_MO_RELAXED
abf9e855	292	#undef RSEQ_TEMPLATE_INDEX_NONE