[librseq.git] / include / rseq / arch / aarch64.h

/* SPDX-License-Identifier: MIT */
/* SPDX-FileCopyrightText: 2016-2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
/* SPDX-FileCopyrightText: 2018 Will Deacon <will.deacon@arm.com> */

/*
 * rseq/arch/aarch64.h
 */

#ifndef _RSEQ_RSEQ_H
#error "Never use <rseq/arch/aarch64.h> directly; include <rseq/rseq.h> instead."
#endif

/*
 * RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those
 * are not part of the public API.
 */

/*
 * aarch64 -mbig-endian generates mixed endianness code vs data:
 * little-endian code and big-endian data. Ensure the RSEQ_SIG signature
 * matches code endianness.
 */
#define RSEQ_SIG_CODE	0xd428bc00	/* BRK #0x45E0.  */

#ifdef __AARCH64EB__	/* Big endian */
# define RSEQ_SIG_DATA	0x00bc28d4	/* BRK #0x45E0.  */
#else			/* Little endian */
# define RSEQ_SIG_DATA	RSEQ_SIG_CODE
#endif

#define RSEQ_SIG	RSEQ_SIG_DATA

/*
 * Refer to the Linux kernel memory model (LKMM) for documentation of
 * the memory barriers.
 */

/* CPU memory barrier. */
#define rseq_smp_mb()	__asm__ __volatile__ ("dmb ish" ::: "memory")
/* CPU read memory barrier */
#define rseq_smp_rmb()	__asm__ __volatile__ ("dmb ishld" ::: "memory")
/* CPU write memory barrier */
#define rseq_smp_wmb()	__asm__ __volatile__ ("dmb ishst" ::: "memory")

/* Acquire: One-way permeable barrier. */
#define rseq_smp_load_acquire(p)						\
__extension__ ({								\
	union { rseq_unqual_scalar_typeof(*(p)) __val; char __c[sizeof(*(p))]; } __u; \
	switch (sizeof(*(p))) {							\
	case 1:									\
		__asm__ __volatile__ ("ldarb %w0, %1"				\
			: "=r" (*(__u8 *)__u.__c)				\
			: "Q" (*(p)) : "memory");				\
		break;								\
	case 2:									\
		__asm__ __volatile__ ("ldarh %w0, %1"				\
			: "=r" (*(__u16 *)__u.__c)				\
			: "Q" (*(p)) : "memory");				\
		break;								\
	case 4:									\
		__asm__ __volatile__ ("ldar %w0, %1"				\
			: "=r" (*(__u32 *)__u.__c)				\
			: "Q" (*(p)) : "memory");				\
		break;								\
	case 8:									\
		__asm__ __volatile__ ("ldar %0, %1"				\
			: "=r" (*(__u64 *)__u.__c)				\
			: "Q" (*(p)) : "memory");				\
		break;								\
	}									\
	(rseq_unqual_scalar_typeof(*(p)))__u.__val;				\
})

/* Acquire barrier after control dependency. */
#define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_rmb()

/* Release: One-way permeable barrier. */
#define rseq_smp_store_release(p, v)						\
do {										\
	union { rseq_unqual_scalar_typeof(*(p)) __val; char __c[sizeof(*(p))]; } __u = \
		{ .__val = (rseq_unqual_scalar_typeof(*(p))) (v) };		\
	switch (sizeof(*(p))) {							\
	case 1:									\
		__asm__ __volatile__ ("stlrb %w1, %0"				\
				: "=Q" (*(p))					\
				: "r" (*(__u8 *)__u.__c)			\
				: "memory");					\
		break;								\
	case 2:									\
		__asm__ __volatile__ ("stlrh %w1, %0"				\
				: "=Q" (*(p))					\
				: "r" (*(__u16 *)__u.__c)			\
				: "memory");					\
		break;								\
	case 4:									\
		__asm__ __volatile__ ("stlr %w1, %0"				\
				: "=Q" (*(p))					\
				: "r" (*(__u32 *)__u.__c)			\
				: "memory");					\
		break;								\
	case 8:									\
		__asm__ __volatile__ ("stlr %1, %0"				\
				: "=Q" (*(p))					\
				: "r" (*(__u64 *)__u.__c)			\
				: "memory");					\
		break;								\
	}									\
} while (0)

#define RSEQ_ASM_U64_PTR(x)	".quad " x
#define RSEQ_ASM_U32(x)		".long " x

/* Temporary scratch registers. */
#define RSEQ_ASM_TMP_REG32	"w15"
#define RSEQ_ASM_TMP_REG	"x15"
#define RSEQ_ASM_TMP_REG_2	"x14"

/* Only used in RSEQ_ASM_DEFINE_TABLE. */
#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip,		\
				post_commit_offset, abort_ip)			\
	"	.pushsection	__rseq_cs, \"aw\"\n"				\
	"	.balign	32\n"							\
	__rseq_str(label) ":\n"							\
	"	" RSEQ_ASM_U32(__rseq_str(version)) "\n"			\
	"	" RSEQ_ASM_U32(__rseq_str(flags)) "\n"				\
	"	" RSEQ_ASM_U64_PTR(__rseq_str(start_ip)) "\n"			\
	"	" RSEQ_ASM_U64_PTR(__rseq_str(post_commit_offset)) "\n"		\
	"	" RSEQ_ASM_U64_PTR(__rseq_str(abort_ip)) "\n"			\
	"	.popsection\n\t"						\
	"	.pushsection __rseq_cs_ptr_array, \"aw\"\n"			\
	"	" RSEQ_ASM_U64_PTR(__rseq_str(label) "b") "\n"			\
	"	.popsection\n"

/*
 * Define an rseq critical section structure of version 0 with no flags.
 *
 *  @label:
 *    Local label for the beginning of the critical section descriptor
 *    structure.
 *  @start_ip:
 *    Pointer to the first instruction of the sequence of consecutive assembly
 *    instructions.
 *  @post_commit_ip:
 *    Pointer to the instruction after the last instruction of the sequence of
 *    consecutive assembly instructions.
 *  @abort_ip:
 *    Pointer to the instruction where to move the execution flow in case of
 *    abort of the sequence of consecutive assembly instructions.
 */
#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip)	\
	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,			\
				(post_commit_ip) - (start_ip), abort_ip)

/*
 * Define the @exit_ip pointer as an exit point for the sequence of consecutive
 * assembly instructions at @start_ip.
 *
 *  @start_ip:
 *    Pointer to the first instruction of the sequence of consecutive assembly
 *    instructions.
 *  @exit_ip:
 *    Pointer to an exit point instruction.
 *
 * Exit points of a rseq critical section consist of all instructions outside
 * of the critical section where a critical section can either branch to or
 * reach through the normal course of its execution. The abort IP and the
 * post-commit IP are already part of the __rseq_cs section and should not be
 * explicitly defined as additional exit points. Knowing all exit points is
 * useful to assist debuggers stepping over the critical section.
 */
#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)				\
	"	.pushsection __rseq_exit_point_array, \"aw\"\n"			\
	"	" RSEQ_ASM_U64_PTR(__rseq_str(start_ip)) "\n"			\
	"	" RSEQ_ASM_U64_PTR(__rseq_str(exit_ip)) "\n"			\
	"	.popsection\n"

/*
 * Define a critical section abort handler.
 *
 *  @label:
 *    Local label to the abort handler.
 *  @teardown:
 *    Sequence of instructions to run on abort.
 *  @abort_label:
 *    C label to jump to at the end of the sequence.
 */
#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)			\
	"	b	222f\n"							\
	"	.inst 	"	__rseq_str(RSEQ_SIG_CODE) "\n"			\
	__rseq_str(label) ":\n"							\
	teardown								\
	"	b	%l[" __rseq_str(abort_label) "]\n"			\
	"222:\n"

/* Jump to local label @label when @cpu_id != @current_cpu_id. */
#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)			\
	RSEQ_INJECT_ASM(1)							\
	"	adrp	" RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n"	\
	"	add	" RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG		\
			", :lo12:" __rseq_str(cs_label) "\n"			\
	"	str	" RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n"	\
	__rseq_str(label) ":\n"

/* Store @value to address @var. */
#define RSEQ_ASM_OP_STORE(value, var)						\
	"	str	%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"

/* Store-release @value to address @var. */
#define RSEQ_ASM_OP_STORE_RELEASE(value, var)					\
	"	stlr	%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"

/*
 * End-of-sequence store of @value to address @var. Emit
 * @post_commit_label label after the store instruction.
 */
#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label)			\
	RSEQ_ASM_OP_STORE(value, var)						\
	__rseq_str(post_commit_label) ":\n"

/*
 * End-of-sequence store-release of @value to address @var. Emit
 * @post_commit_label label after the store instruction.
 */
#define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label)		\
	RSEQ_ASM_OP_STORE_RELEASE(value, var)					\
	__rseq_str(post_commit_label) ":\n"

/* Jump to local label @label when @var != @expect. */
#define RSEQ_ASM_OP_CBNE(var, expect, label)					\
	"	ldr	" RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"		\
	"	sub	" RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG		\
			", %[" __rseq_str(expect) "]\n"				\
	"	cbnz	" RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"

/*
 * Jump to local label @label when @var != @expect (32-bit register
 * comparison).
 */
#define RSEQ_ASM_OP_CBNE32(var, expect, label)					\
	"	ldr	" RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n"	\
	"	sub	" RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32		\
			", %w[" __rseq_str(expect) "]\n"			\
	"	cbnz	" RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n"

/* Jump to local label @label when @var == @expect. */
#define RSEQ_ASM_OP_CBEQ(var, expect, label)					\
	"	ldr	" RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"		\
	"	sub	" RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG		\
			", %[" __rseq_str(expect) "]\n"				\
	"	cbz	" RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"

/* Jump to local label @label when @cpu_id != @current_cpu_id. */
#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label)			\
	RSEQ_INJECT_ASM(2)							\
	RSEQ_ASM_OP_CBNE32(current_cpu_id, cpu_id, label)

/* Load @var into temporary register. */
#define RSEQ_ASM_OP_R_LOAD(var)							\
	"	ldr	" RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"

/* Store from temporary register into @var. */
#define RSEQ_ASM_OP_R_STORE(var)						\
	"	str	" RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"

/* Load from address in temporary register+@offset into temporary register. */
#define RSEQ_ASM_OP_R_LOAD_OFF(offset)						\
	"	ldr	" RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG		\
			", %[" __rseq_str(offset) "]]\n"

/* Add @count to temporary register. */
#define RSEQ_ASM_OP_R_ADD(count)						\
	"	add	" RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG		\
			", %[" __rseq_str(count) "]\n"

/*
 * End-of-sequence store of temporary register to address @var. Emit
 * @post_commit_label label after the store instruction.
 */
#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label)			\
	"	str	" RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"		\
	__rseq_str(post_commit_label) ":\n"

/*
 * Copy @len bytes from @src to @dst. This is an inefficient bytewise
 * copy and could be improved in the future.
 */
#define RSEQ_ASM_OP_R_BYTEWISE_MEMCPY(dst, src, len)				\
	"	cbz	%[" __rseq_str(len) "], 333f\n"				\
	"	mov	" RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n"	\
	"222:	sub	" RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n"	\
	"	ldrb	" RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]"	\
			", " RSEQ_ASM_TMP_REG_2 "]\n"				\
	"	strb	" RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]"	\
			", " RSEQ_ASM_TMP_REG_2 "]\n"				\
	"	cbnz	" RSEQ_ASM_TMP_REG_2 ", 222b\n"				\
	"333:\n"

/* Per-cpu-id indexing. */

#define RSEQ_TEMPLATE_INDEX_CPU_ID
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq/arch/aarch64/bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED

#define RSEQ_TEMPLATE_MO_RELEASE
#include "rseq/arch/aarch64/bits.h"
#undef RSEQ_TEMPLATE_MO_RELEASE
#undef RSEQ_TEMPLATE_INDEX_CPU_ID

/* Per-mm-cid indexing. */

#define RSEQ_TEMPLATE_INDEX_MM_CID
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq/arch/aarch64/bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED

#define RSEQ_TEMPLATE_MO_RELEASE
#include "rseq/arch/aarch64/bits.h"
#undef RSEQ_TEMPLATE_MO_RELEASE
#undef RSEQ_TEMPLATE_INDEX_MM_CID

/* APIs which are not indexed. */

#define RSEQ_TEMPLATE_INDEX_NONE
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq/arch/aarch64/bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED
#undef RSEQ_TEMPLATE_INDEX_NONE
Commit	Line	Data
90702366	1	/* SPDX-License-Identifier: MIT */
21b58a3b	2	/* SPDX-FileCopyrightText: 2016-2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
f2d7b530 MJ	3	/* SPDX-FileCopyrightText: 2018 Will Deacon <will.deacon@arm.com> */
f2d7b530 MJ	4
d78a16c2	5	/*
44ec21eb	6	* rseq/arch/aarch64.h
d78a16c2 MD	7	*/
d78a16c2 MD	8
44ec21eb MJ	9	#ifndef _RSEQ_RSEQ_H
	10	#error "Never use <rseq/arch/aarch64.h> directly; include <rseq/rseq.h> instead."
	11	#endif
	12
21b58a3b MD	13	/*
	14	* RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those
	15	* are not part of the public API.
	16	*/
	17
8b7c64f7 MD	18	/*
	19	* aarch64 -mbig-endian generates mixed endianness code vs data:
	20	* little-endian code and big-endian data. Ensure the RSEQ_SIG signature
	21	* matches code endianness.
	22	*/
	23	#define RSEQ_SIG_CODE 0xd428bc00 /* BRK #0x45E0. */
	24
dbbd006a MD	25	#ifdef __AARCH64EB__ /* Big endian */
	26	# define RSEQ_SIG_DATA 0x00bc28d4 /* BRK #0x45E0. */
	27	#else /* Little endian */
	28	# define RSEQ_SIG_DATA RSEQ_SIG_CODE
8b7c64f7 MD	29	#endif
	30
	31	#define RSEQ_SIG RSEQ_SIG_DATA
d78a16c2	32
21b58a3b MD	33	/*
	34	* Refer to the Linux kernel memory model (LKMM) for documentation of
	35	* the memory barriers.
	36	*/
	37
	38	/* CPU memory barrier. */
d78a16c2	39	#define rseq_smp_mb() __asm__ __volatile__ ("dmb ish" ::: "memory")
21b58a3b	40	/* CPU read memory barrier */
d78a16c2	41	#define rseq_smp_rmb() __asm__ __volatile__ ("dmb ishld" ::: "memory")
21b58a3b	42	/* CPU write memory barrier */
d78a16c2 MD	43	#define rseq_smp_wmb() __asm__ __volatile__ ("dmb ishst" ::: "memory")
d78a16c2 MD	44
21b58a3b	45	/* Acquire: One-way permeable barrier. */
d78a16c2 MD	46	#define rseq_smp_load_acquire(p) \
d78a16c2 MD	47	__extension__ ({ \
18ca65e0 MD	48	union { rseq_unqual_scalar_typeof((p)) __val; char __c[sizeof((p))]; } __u; \
18ca65e0 MD	49	switch (sizeof(*(p))) { \
d78a16c2	50	case 1: \
18ca65e0 MD	51	__asm__ __volatile__ ("ldarb %w0, %1" \
	52	: "=r" ((__u8 )__u.__c) \
	53	: "Q" (*(p)) : "memory"); \
d78a16c2 MD	54	break; \
d78a16c2 MD	55	case 2: \
18ca65e0 MD	56	__asm__ __volatile__ ("ldarh %w0, %1" \
	57	: "=r" ((__u16 )__u.__c) \
	58	: "Q" (*(p)) : "memory"); \
d78a16c2 MD	59	break; \
d78a16c2 MD	60	case 4: \
18ca65e0 MD	61	__asm__ __volatile__ ("ldar %w0, %1" \
	62	: "=r" ((__u32 )__u.__c) \
	63	: "Q" (*(p)) : "memory"); \
d78a16c2 MD	64	break; \
d78a16c2 MD	65	case 8: \
18ca65e0 MD	66	__asm__ __volatile__ ("ldar %0, %1" \
	67	: "=r" ((__u64 )__u.__c) \
	68	: "Q" (*(p)) : "memory"); \
d78a16c2 MD	69	break; \
d78a16c2 MD	70	} \
0aac367c	71	(rseq_unqual_scalar_typeof(*(p)))__u.__val; \
d78a16c2 MD	72	})
d78a16c2 MD	73
21b58a3b	74	/* Acquire barrier after control dependency. */
d78a16c2 MD	75	#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
d78a16c2 MD	76
21b58a3b	77	/* Release: One-way permeable barrier. */
d78a16c2 MD	78	#define rseq_smp_store_release(p, v) \
d78a16c2 MD	79	do { \
18ca65e0 MD	80	union { rseq_unqual_scalar_typeof((p)) __val; char __c[sizeof((p))]; } __u = \
	81	{ .__val = (rseq_unqual_scalar_typeof(*(p))) (v) }; \
	82	switch (sizeof(*(p))) { \
d78a16c2	83	case 1: \
18ca65e0 MD	84	__asm__ __volatile__ ("stlrb %w1, %0" \
	85	: "=Q" (*(p)) \
	86	: "r" ((__u8 )__u.__c) \
d78a16c2 MD	87	: "memory"); \
	88	break; \
	89	case 2: \
18ca65e0 MD	90	__asm__ __volatile__ ("stlrh %w1, %0" \
	91	: "=Q" (*(p)) \
	92	: "r" ((__u16 )__u.__c) \
d78a16c2 MD	93	: "memory"); \
	94	break; \
	95	case 4: \
18ca65e0 MD	96	__asm__ __volatile__ ("stlr %w1, %0" \
	97	: "=Q" (*(p)) \
	98	: "r" ((__u32 )__u.__c) \
d78a16c2 MD	99	: "memory"); \
	100	break; \
	101	case 8: \
18ca65e0 MD	102	__asm__ __volatile__ ("stlr %1, %0" \
	103	: "=Q" (*(p)) \
	104	: "r" ((__u64 )__u.__c) \
d78a16c2 MD	105	: "memory"); \
	106	break; \
	107	} \
	108	} while (0)
	109
9ed48921	110	#define RSEQ_ASM_U64_PTR(x) ".quad " x
0f986786	111	#define RSEQ_ASM_U32(x) ".long " x
9ed48921	112
21b58a3b	113	/* Temporary scratch registers. */
d78a16c2 MD	114	#define RSEQ_ASM_TMP_REG32 "w15"
	115	#define RSEQ_ASM_TMP_REG "x15"
	116	#define RSEQ_ASM_TMP_REG_2 "x14"
	117
21b58a3b	118	/* Only used in RSEQ_ASM_DEFINE_TABLE. */
d78a16c2 MD	119	#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
d78a16c2 MD	120	post_commit_offset, abort_ip) \
dd01d0fb	121	" .pushsection __rseq_cs, \"aw\"\n" \
d78a16c2 MD	122	" .balign 32\n" \
d78a16c2 MD	123	__rseq_str(label) ":\n" \
0f986786 MD	124	" " RSEQ_ASM_U32(__rseq_str(version)) "\n" \
0f986786 MD	125	" " RSEQ_ASM_U32(__rseq_str(flags)) "\n" \
9ed48921 MD	126	" " RSEQ_ASM_U64_PTR(__rseq_str(start_ip)) "\n" \
	127	" " RSEQ_ASM_U64_PTR(__rseq_str(post_commit_offset)) "\n" \
	128	" " RSEQ_ASM_U64_PTR(__rseq_str(abort_ip)) "\n" \
dd01d0fb	129	" .popsection\n\t" \
21b58a3b	130	" .pushsection __rseq_cs_ptr_array, \"aw\"\n" \
9ed48921	131	" " RSEQ_ASM_U64_PTR(__rseq_str(label) "b") "\n" \
d78a16c2 MD	132	" .popsection\n"
d78a16c2 MD	133
21b58a3b MD	134	/*
	135	* Define an rseq critical section structure of version 0 with no flags.
	136	*
	137	* @label:
	138	* Local label for the beginning of the critical section descriptor
	139	* structure.
	140	* @start_ip:
	141	* Pointer to the first instruction of the sequence of consecutive assembly
	142	* instructions.
	143	* @post_commit_ip:
	144	* Pointer to the instruction after the last instruction of the sequence of
	145	* consecutive assembly instructions.
	146	* @abort_ip:
	147	* Pointer to the instruction where to move the execution flow in case of
	148	* abort of the sequence of consecutive assembly instructions.
	149	*/
d78a16c2 MD	150	#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
d78a16c2 MD	151	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
d15481d2	152	(post_commit_ip) - (start_ip), abort_ip)
d78a16c2	153
90d9876e	154	/*
21b58a3b MD	155	* Define the @exit_ip pointer as an exit point for the sequence of consecutive
	156	* assembly instructions at @start_ip.
	157	*
	158	* @start_ip:
	159	* Pointer to the first instruction of the sequence of consecutive assembly
	160	* instructions.
	161	* @exit_ip:
	162	* Pointer to an exit point instruction.
	163	*
90d9876e	164	* Exit points of a rseq critical section consist of all instructions outside
fd622cad MD	165	* of the critical section where a critical section can either branch to or
	166	* reach through the normal course of its execution. The abort IP and the
	167	* post-commit IP are already part of the __rseq_cs section and should not be
	168	* explicitly defined as additional exit points. Knowing all exit points is
	169	* useful to assist debuggers stepping over the critical section.
90d9876e MD	170	*/
	171	#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
	172	" .pushsection __rseq_exit_point_array, \"aw\"\n" \
9ed48921 MD	173	" " RSEQ_ASM_U64_PTR(__rseq_str(start_ip)) "\n" \
9ed48921 MD	174	" " RSEQ_ASM_U64_PTR(__rseq_str(exit_ip)) "\n" \
90d9876e MD	175	" .popsection\n"
90d9876e MD	176
21b58a3b MD	177	/*
	178	* Define a critical section abort handler.
	179	*
	180	* @label:
	181	* Local label to the abort handler.
	182	* @teardown:
	183	* Sequence of instructions to run on abort.
	184	* @abort_label:
	185	* C label to jump to at the end of the sequence.
	186	*/
ebd27573	187	#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
d78a16c2	188	" b 222f\n" \
8b7c64f7	189	" .inst " __rseq_str(RSEQ_SIG_CODE) "\n" \
d78a16c2	190	__rseq_str(label) ":\n" \
ebd27573	191	teardown \
d78a16c2 MD	192	" b %l[" __rseq_str(abort_label) "]\n" \
	193	"222:\n"
	194
21b58a3b MD	195	/* Jump to local label @label when @cpu_id != @current_cpu_id. */
	196	#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
	197	RSEQ_INJECT_ASM(1) \
	198	" adrp " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n" \
	199	" add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
	200	", :lo12:" __rseq_str(cs_label) "\n" \
	201	" str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n" \
	202	__rseq_str(label) ":\n"
	203
	204	/* Store @value to address @var. */
d78a16c2 MD	205	#define RSEQ_ASM_OP_STORE(value, var) \
	206	" str %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
	207
21b58a3b	208	/* Store-release @value to address @var. */
d78a16c2 MD	209	#define RSEQ_ASM_OP_STORE_RELEASE(value, var) \
	210	" stlr %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
	211
21b58a3b MD	212	/*
	213	* End-of-sequence store of @value to address @var. Emit
	214	* @post_commit_label label after the store instruction.
	215	*/
d78a16c2 MD	216	#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
	217	RSEQ_ASM_OP_STORE(value, var) \
	218	__rseq_str(post_commit_label) ":\n"
	219
21b58a3b MD	220	/*
	221	* End-of-sequence store-release of @value to address @var. Emit
	222	* @post_commit_label label after the store instruction.
	223	*/
d78a16c2 MD	224	#define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \
	225	RSEQ_ASM_OP_STORE_RELEASE(value, var) \
	226	__rseq_str(post_commit_label) ":\n"
	227
21b58a3b	228	/* Jump to local label @label when @var != @expect. */
769ec9a5	229	#define RSEQ_ASM_OP_CBNE(var, expect, label) \
d78a16c2 MD	230	" ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
	231	" sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
	232	", %[" __rseq_str(expect) "]\n" \
	233	" cbnz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
	234
21b58a3b MD	235	/*
	236	* Jump to local label @label when @var != @expect (32-bit register
	237	* comparison).
	238	*/
769ec9a5	239	#define RSEQ_ASM_OP_CBNE32(var, expect, label) \
d78a16c2 MD	240	" ldr " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n" \
	241	" sub " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32 \
	242	", %w[" __rseq_str(expect) "]\n" \
	243	" cbnz " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n"
	244
21b58a3b	245	/* Jump to local label @label when @var == @expect. */
769ec9a5	246	#define RSEQ_ASM_OP_CBEQ(var, expect, label) \
d78a16c2 MD	247	" ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
	248	" sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
	249	", %[" __rseq_str(expect) "]\n" \
	250	" cbz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
	251
21b58a3b	252	/* Jump to local label @label when @cpu_id != @current_cpu_id. */
769ec9a5	253	#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \
d78a16c2	254	RSEQ_INJECT_ASM(2) \
769ec9a5	255	RSEQ_ASM_OP_CBNE32(current_cpu_id, cpu_id, label)
d78a16c2	256
21b58a3b	257	/* Load @var into temporary register. */
d78a16c2 MD	258	#define RSEQ_ASM_OP_R_LOAD(var) \
	259	" ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
	260
21b58a3b	261	/* Store from temporary register into @var. */
d78a16c2 MD	262	#define RSEQ_ASM_OP_R_STORE(var) \
	263	" str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
	264
21b58a3b	265	/* Load from address in temporary register+@offset into temporary register. */
d78a16c2 MD	266	#define RSEQ_ASM_OP_R_LOAD_OFF(offset) \
	267	" ldr " RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG \
	268	", %[" __rseq_str(offset) "]]\n"
	269
21b58a3b	270	/* Add @count to temporary register. */
d78a16c2 MD	271	#define RSEQ_ASM_OP_R_ADD(count) \
	272	" add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
	273	", %[" __rseq_str(count) "]\n"
	274
21b58a3b MD	275	/*
	276	* End-of-sequence store of temporary register to address @var. Emit
	277	* @post_commit_label label after the store instruction.
	278	*/
d78a16c2 MD	279	#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
	280	" str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
	281	__rseq_str(post_commit_label) ":\n"
	282
21b58a3b MD	283	/*
	284	* Copy @len bytes from @src to @dst. This is an inefficient bytewise
	285	* copy and could be improved in the future.
	286	*/
6020ff66	287	#define RSEQ_ASM_OP_R_BYTEWISE_MEMCPY(dst, src, len) \
d78a16c2 MD	288	" cbz %[" __rseq_str(len) "], 333f\n" \
	289	" mov " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n" \
	290	"222: sub " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n" \
	291	" ldrb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]" \
	292	", " RSEQ_ASM_TMP_REG_2 "]\n" \
	293	" strb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]" \
	294	", " RSEQ_ASM_TMP_REG_2 "]\n" \
	295	" cbnz " RSEQ_ASM_TMP_REG_2 ", 222b\n" \
	296	"333:\n"
	297
a98a3b6d	298	/* Per-cpu-id indexing. */
d78a16c2	299
abf9e855	300	#define RSEQ_TEMPLATE_INDEX_CPU_ID
a98a3b6d	301	#define RSEQ_TEMPLATE_MO_RELAXED
44ec21eb	302	#include "rseq/arch/aarch64/bits.h"
a98a3b6d	303	#undef RSEQ_TEMPLATE_MO_RELAXED
d78a16c2	304
a98a3b6d	305	#define RSEQ_TEMPLATE_MO_RELEASE
44ec21eb	306	#include "rseq/arch/aarch64/bits.h"
a98a3b6d	307	#undef RSEQ_TEMPLATE_MO_RELEASE
abf9e855	308	#undef RSEQ_TEMPLATE_INDEX_CPU_ID
a98a3b6d MD	309
	310	/* Per-mm-cid indexing. */
	311
abf9e855	312	#define RSEQ_TEMPLATE_INDEX_MM_CID
a98a3b6d	313	#define RSEQ_TEMPLATE_MO_RELAXED
44ec21eb	314	#include "rseq/arch/aarch64/bits.h"
a98a3b6d MD	315	#undef RSEQ_TEMPLATE_MO_RELAXED
	316
	317	#define RSEQ_TEMPLATE_MO_RELEASE
44ec21eb	318	#include "rseq/arch/aarch64/bits.h"
a98a3b6d	319	#undef RSEQ_TEMPLATE_MO_RELEASE
abf9e855	320	#undef RSEQ_TEMPLATE_INDEX_MM_CID
a98a3b6d	321
abf9e855	322	/* APIs which are not indexed. */
a98a3b6d	323
abf9e855	324	#define RSEQ_TEMPLATE_INDEX_NONE
a98a3b6d	325	#define RSEQ_TEMPLATE_MO_RELAXED
44ec21eb	326	#include "rseq/arch/aarch64/bits.h"
a98a3b6d	327	#undef RSEQ_TEMPLATE_MO_RELAXED
abf9e855	328	#undef RSEQ_TEMPLATE_INDEX_NONE