[librseq.git] / include / rseq / arch / arm.h

/* SPDX-License-Identifier: MIT */
/* SPDX-FileCopyrightText: 2016-2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */

/*
 * rseq/arch/arm.h
 */

#ifndef _RSEQ_RSEQ_H
#error "Never use <rseq/arch/arm.h> directly; include <rseq/rseq.h> instead."
#endif

/*
 * RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those
 * are not part of the public API.
 */

/*
 * - ARM little endian
 *
 * RSEQ_SIG uses the udf A32 instruction with an uncommon immediate operand
 * value 0x5de3. This traps if user-space reaches this instruction by mistake,
 * and the uncommon operand ensures the kernel does not move the instruction
 * pointer to attacker-controlled code on rseq abort.
 *
 * The instruction pattern in the A32 instruction set is:
 *
 * e7f5def3    udf    #24035    ; 0x5de3
 *
 * This translates to the following instruction pattern in the T16 instruction
 * set:
 *
 * little endian:
 * def3        udf    #243      ; 0xf3
 * e7f5        b.n    <7f5>
 *
 * - ARMv6+ big endian (BE8):
 *
 * ARMv6+ -mbig-endian generates mixed endianness code vs data: little-endian
 * code and big-endian data. The data value of the signature needs to have its
 * byte order reversed to generate the trap instruction:
 *
 * Data: 0xf3def5e7
 *
 * Translates to this A32 instruction pattern:
 *
 * e7f5def3    udf    #24035    ; 0x5de3
 *
 * Translates to this T16 instruction pattern:
 *
 * def3        udf    #243      ; 0xf3
 * e7f5        b.n    <7f5>
 *
 * - Prior to ARMv6 big endian (BE32):
 *
 * Prior to ARMv6, -mbig-endian generates big-endian code and data
 * (which match), so the endianness of the data representation of the
 * signature should not be reversed. However, the choice between BE32
 * and BE8 is done by the linker, so we cannot know whether code and
 * data endianness will be mixed before the linker is invoked. So rather
 * than try to play tricks with the linker, the rseq signature is simply
 * data (not a trap instruction) prior to ARMv6 on big endian. This is
 * why the signature is expressed as data (.word) rather than as
 * instruction (.inst) in assembler.
 */

#ifdef __ARMEB__	/* Big endian */
# define RSEQ_SIG	0xf3def5e7	/* udf    #24035    ; 0x5de3 (ARMv6+) */
#else			/* Little endian */
# define RSEQ_SIG	0xe7f5def3	/* udf    #24035    ; 0x5de3 */
#endif

/*
 * Refer to the Linux kernel memory model (LKMM) for documentation of
 * the memory barriers.
 */

/* CPU memory barrier. */
#define rseq_smp_mb()	__asm__ __volatile__ ("dmb" ::: "memory", "cc")
/* CPU read memory barrier */
#define rseq_smp_rmb()	__asm__ __volatile__ ("dmb" ::: "memory", "cc")
/* CPU write memory barrier */
#define rseq_smp_wmb()	__asm__ __volatile__ ("dmb" ::: "memory", "cc")

/* Acquire: One-way permeable barrier. */
#define rseq_smp_load_acquire(p)					\
__extension__ ({							\
	rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p));	\
	rseq_smp_mb();							\
	____p1;								\
})

/* Acquire barrier after control dependency. */
#define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_rmb()

/* Release: One-way permeable barrier. */
#define rseq_smp_store_release(p, v)					\
do {									\
	rseq_smp_mb();							\
	RSEQ_WRITE_ONCE(*(p), v);					\
} while (0)

#ifdef __ARMEB__	/* Big endian */
# define RSEQ_ASM_U64_PTR(x)		".word 0x0, " x
#else			/* Little endian */
# define RSEQ_ASM_U64_PTR(x)		".word " x ", 0x0"
#endif

#define RSEQ_ASM_U32(x)			".word " x

/* Common architecture support macros. */
#include "rseq/arch/generic/common.h"

/*
 * Store the address of the critical section descriptor structure at
 * @cs_label into the @rseq_cs pointer and emit the label @label, which
 * is the beginning of the sequence of consecutive assembly instructions.
 *
 *  @label:
 *    Local label to the beginning of the sequence of consecutive assembly
 *    instructions.
 *  @cs_label:
 *    Source local label to the critical section descriptor structure.
 *  @rseq_cs:
 *    Destination pointer where to store the address of the critical
 *    section descriptor structure.
 */
#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)		\
		RSEQ_INJECT_ASM(1)					\
		"adr r0, " __rseq_str(cs_label) "\n\t"			\
		"str r0, %[" __rseq_str(rseq_cs) "]\n\t"		\
		__rseq_str(label) ":\n\t"

/* Only used in RSEQ_ASM_DEFINE_ABORT.  */
#define __RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label,		\
				table_label, version, flags,		\
				start_ip, post_commit_offset, abort_ip)	\
		".balign 32\n\t"					\
		__rseq_str(table_label) ":\n\t"				\
		RSEQ_ASM_U32(__rseq_str(version)) "\n\t" 		\
		RSEQ_ASM_U32(__rseq_str(flags)) "\n\t" 			\
		RSEQ_ASM_U64_PTR(__rseq_str(start_ip)) "\n\t"		\
		RSEQ_ASM_U64_PTR(__rseq_str(post_commit_offset)) "\n\t" \
		RSEQ_ASM_U64_PTR(__rseq_str(abort_ip)) "\n\t"		\
		RSEQ_ASM_U32(__rseq_str(RSEQ_SIG)) "\n\t"		\
		__rseq_str(label) ":\n\t"				\
		teardown						\
		"b %l[" __rseq_str(abort_label) "]\n\t"

/*
 * Define a critical section abort handler.
 *
 *  @label:
 *    Local label to the abort handler.
 *  @teardown:
 *    Sequence of instructions to run on abort.
 *  @abort_label:
 *    C label to jump to at the end of the sequence.
 *  @table_label:
 *    Local label to the critical section descriptor copy placed near
 *    the program counter. This is done for performance reasons because
 *    computing this address is faster than accessing the program data.
 *
 * The purpose of @start_ip, @post_commit_ip, and @abort_ip are
 * documented in RSEQ_ASM_DEFINE_TABLE.
 */
#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label,		\
			      table_label, start_ip, post_commit_ip, abort_ip) \
	__RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label,		\
				table_label, 0x0, 0x0, start_ip,	\
				(post_commit_ip - start_ip), abort_ip)

/*
 * Define a critical section teardown handler.
 *
 *  @label:
 *    Local label to the teardown handler.
 *  @teardown:
 *    Sequence of instructions to run on teardown.
 *  @target_label:
 *    C label to jump to at the end of the sequence.
 */
#define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label)		\
		__rseq_str(label) ":\n\t"				\
		teardown						\
		"b %l[" __rseq_str(target_label) "]\n\t"

/* Jump to local label @label when @cpu_id != @current_cpu_id. */
#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label)		\
		RSEQ_INJECT_ASM(2)					\
		"ldr r0, %[" __rseq_str(current_cpu_id) "]\n\t"		\
		"cmp %[" __rseq_str(cpu_id) "], r0\n\t"			\
		"bne " __rseq_str(label) "\n\t"

/* Per-cpu-id indexing. */

#define RSEQ_TEMPLATE_INDEX_CPU_ID
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq/arch/arm/bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED

#define RSEQ_TEMPLATE_MO_RELEASE
#include "rseq/arch/arm/bits.h"
#undef RSEQ_TEMPLATE_MO_RELEASE
#undef RSEQ_TEMPLATE_INDEX_CPU_ID

/* Per-mm-cid indexing. */

#define RSEQ_TEMPLATE_INDEX_MM_CID
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq/arch/arm/bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED

#define RSEQ_TEMPLATE_MO_RELEASE
#include "rseq/arch/arm/bits.h"
#undef RSEQ_TEMPLATE_MO_RELEASE
#undef RSEQ_TEMPLATE_INDEX_MM_CID

/* APIs which are not indexed. */

#define RSEQ_TEMPLATE_INDEX_NONE
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq/arch/arm/bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED
#undef RSEQ_TEMPLATE_INDEX_NONE
Commit	Line	Data
90702366	1	/* SPDX-License-Identifier: MIT */
1bb8dd7b	2	/* SPDX-FileCopyrightText: 2016-2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
f2d7b530	3
784b0012	4	/*
44ec21eb	5	* rseq/arch/arm.h
784b0012 MD	6	*/
784b0012 MD	7
44ec21eb MJ	8	#ifndef _RSEQ_RSEQ_H
	9	#error "Never use <rseq/arch/arm.h> directly; include <rseq/rseq.h> instead."
	10	#endif
	11
1bb8dd7b MD	12	/*
	13	* RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those
	14	* are not part of the public API.
	15	*/
	16
40903c88	17	/*
4b5bf705 MD	18	* - ARM little endian
4b5bf705 MD	19	*
40903c88 MD	20	* RSEQ_SIG uses the udf A32 instruction with an uncommon immediate operand
	21	* value 0x5de3. This traps if user-space reaches this instruction by mistake,
	22	* and the uncommon operand ensures the kernel does not move the instruction
	23	* pointer to attacker-controlled code on rseq abort.
	24	*
	25	* The instruction pattern in the A32 instruction set is:
	26	*
	27	* e7f5def3 udf #24035 ; 0x5de3
	28	*
	29	* This translates to the following instruction pattern in the T16 instruction
	30	* set:
	31	*
	32	* little endian:
	33	* def3 udf #243 ; 0xf3
	34	* e7f5 b.n <7f5>
	35	*
4b5bf705	36	* - ARMv6+ big endian (BE8):
40903c88 MD	37	*
40903c88 MD	38	* ARMv6+ -mbig-endian generates mixed endianness code vs data: little-endian
4b5bf705 MD	39	* code and big-endian data. The data value of the signature needs to have its
	40	* byte order reversed to generate the trap instruction:
	41	*
	42	* Data: 0xf3def5e7
	43	*
	44	* Translates to this A32 instruction pattern:
	45	*
	46	* e7f5def3 udf #24035 ; 0x5de3
	47	*
	48	* Translates to this T16 instruction pattern:
	49	*
	50	* def3 udf #243 ; 0xf3
	51	* e7f5 b.n <7f5>
	52	*
	53	* - Prior to ARMv6 big endian (BE32):
	54	*
	55	* Prior to ARMv6, -mbig-endian generates big-endian code and data
	56	* (which match), so the endianness of the data representation of the
	57	* signature should not be reversed. However, the choice between BE32
	58	* and BE8 is done by the linker, so we cannot know whether code and
	59	* data endianness will be mixed before the linker is invoked. So rather
	60	* than try to play tricks with the linker, the rseq signature is simply
	61	* data (not a trap instruction) prior to ARMv6 on big endian. This is
	62	* why the signature is expressed as data (.word) rather than as
	63	* instruction (.inst) in assembler.
40903c88 MD	64	*/
40903c88 MD	65
ff36aa74 MD	66	#ifdef __ARMEB__ /* Big endian */
	67	# define RSEQ_SIG 0xf3def5e7 /* udf #24035 ; 0x5de3 (ARMv6+) */
	68	#else /* Little endian */
	69	# define RSEQ_SIG 0xe7f5def3 /* udf #24035 ; 0x5de3 */
40903c88	70	#endif
784b0012	71
1bb8dd7b MD	72	/*
	73	* Refer to the Linux kernel memory model (LKMM) for documentation of
	74	* the memory barriers.
	75	*/
	76
	77	/* CPU memory barrier. */
784b0012	78	#define rseq_smp_mb() __asm__ __volatile__ ("dmb" ::: "memory", "cc")
1bb8dd7b	79	/* CPU read memory barrier */
784b0012	80	#define rseq_smp_rmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc")
1bb8dd7b	81	/* CPU write memory barrier */
784b0012 MD	82	#define rseq_smp_wmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc")
784b0012 MD	83
1bb8dd7b	84	/* Acquire: One-way permeable barrier. */
784b0012 MD	85	#define rseq_smp_load_acquire(p) \
784b0012 MD	86	__extension__ ({ \
3664a718	87	rseq_unqual_scalar_typeof((p)) ____p1 = RSEQ_READ_ONCE((p)); \
784b0012 MD	88	rseq_smp_mb(); \
	89	____p1; \
	90	})
	91
1bb8dd7b	92	/* Acquire barrier after control dependency. */
784b0012 MD	93	#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
784b0012 MD	94
1bb8dd7b	95	/* Release: One-way permeable barrier. */
784b0012 MD	96	#define rseq_smp_store_release(p, v) \
	97	do { \
	98	rseq_smp_mb(); \
826417f6	99	RSEQ_WRITE_ONCE(*(p), v); \
784b0012 MD	100	} while (0)
784b0012 MD	101
4cb890a6 MD	102	#ifdef __ARMEB__ /* Big endian */
	103	# define RSEQ_ASM_U64_PTR(x) ".word 0x0, " x
	104	#else /* Little endian */
	105	# define RSEQ_ASM_U64_PTR(x) ".word " x ", 0x0"
	106	#endif
	107
500b4c05 MD	108	#define RSEQ_ASM_U32(x) ".word " x
500b4c05 MD	109
ed21bf6d MD	110	/* Common architecture support macros. */
ed21bf6d MD	111	#include "rseq/arch/generic/common.h"
90d9876e	112
1bb8dd7b MD	113	/*
	114	* Store the address of the critical section descriptor structure at
	115	* @cs_label into the @rseq_cs pointer and emit the label @label, which
	116	* is the beginning of the sequence of consecutive assembly instructions.
	117	*
	118	* @label:
	119	* Local label to the beginning of the sequence of consecutive assembly
	120	* instructions.
	121	* @cs_label:
	122	* Source local label to the critical section descriptor structure.
	123	* @rseq_cs:
	124	* Destination pointer where to store the address of the critical
	125	* section descriptor structure.
	126	*/
784b0012 MD	127	#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
	128	RSEQ_INJECT_ASM(1) \
	129	"adr r0, " __rseq_str(cs_label) "\n\t" \
	130	"str r0, %[" __rseq_str(rseq_cs) "]\n\t" \
	131	__rseq_str(label) ":\n\t"
	132
1bb8dd7b	133	/* Only used in RSEQ_ASM_DEFINE_ABORT. */
1bb8c5e3 MD	134	#define __RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label, \
1bb8c5e3 MD	135	table_label, version, flags, \
784b0012 MD	136	start_ip, post_commit_offset, abort_ip) \
	137	".balign 32\n\t" \
	138	__rseq_str(table_label) ":\n\t" \
500b4c05 MD	139	RSEQ_ASM_U32(__rseq_str(version)) "\n\t" \
500b4c05 MD	140	RSEQ_ASM_U32(__rseq_str(flags)) "\n\t" \
4cb890a6 MD	141	RSEQ_ASM_U64_PTR(__rseq_str(start_ip)) "\n\t" \
	142	RSEQ_ASM_U64_PTR(__rseq_str(post_commit_offset)) "\n\t" \
	143	RSEQ_ASM_U64_PTR(__rseq_str(abort_ip)) "\n\t" \
500b4c05	144	RSEQ_ASM_U32(__rseq_str(RSEQ_SIG)) "\n\t" \
784b0012 MD	145	__rseq_str(label) ":\n\t" \
	146	teardown \
	147	"b %l[" __rseq_str(abort_label) "]\n\t"
	148
1bb8dd7b MD	149	/*
	150	* Define a critical section abort handler.
	151	*
1bb8dd7b MD	152	* @label:
	153	* Local label to the abort handler.
	154	* @teardown:
	155	* Sequence of instructions to run on abort.
	156	* @abort_label:
	157	* C label to jump to at the end of the sequence.
1bb8c5e3 MD	158	* @table_label:
	159	* Local label to the critical section descriptor copy placed near
	160	* the program counter. This is done for performance reasons because
	161	* computing this address is faster than accessing the program data.
1bb8dd7b MD	162	*
	163	* The purpose of @start_ip, @post_commit_ip, and @abort_ip are
	164	* documented in RSEQ_ASM_DEFINE_TABLE.
	165	*/
1bb8c5e3 MD	166	#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label, \
	167	table_label, start_ip, post_commit_ip, abort_ip) \
	168	__RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label, \
	169	table_label, 0x0, 0x0, start_ip, \
784b0012 MD	170	(post_commit_ip - start_ip), abort_ip)
784b0012 MD	171
1bb8dd7b MD	172	/*
	173	* Define a critical section teardown handler.
	174	*
	175	* @label:
	176	* Local label to the teardown handler.
	177	* @teardown:
	178	* Sequence of instructions to run on teardown.
	179	* @target_label:
	180	* C label to jump to at the end of the sequence.
	181	*/
559d824f	182	#define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label) \
784b0012 MD	183	__rseq_str(label) ":\n\t" \
784b0012 MD	184	teardown \
559d824f	185	"b %l[" __rseq_str(target_label) "]\n\t"
784b0012	186
1bb8dd7b MD	187	/* Jump to local label @label when @cpu_id != @current_cpu_id. */
	188	#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \
	189	RSEQ_INJECT_ASM(2) \
	190	"ldr r0, %[" __rseq_str(current_cpu_id) "]\n\t" \
	191	"cmp %[" __rseq_str(cpu_id) "], r0\n\t" \
	192	"bne " __rseq_str(label) "\n\t"
	193
a66de583	194	/* Per-cpu-id indexing. */
784b0012	195
abf9e855	196	#define RSEQ_TEMPLATE_INDEX_CPU_ID
a66de583	197	#define RSEQ_TEMPLATE_MO_RELAXED
44ec21eb	198	#include "rseq/arch/arm/bits.h"
a66de583	199	#undef RSEQ_TEMPLATE_MO_RELAXED
784b0012	200
a66de583	201	#define RSEQ_TEMPLATE_MO_RELEASE
44ec21eb	202	#include "rseq/arch/arm/bits.h"
a66de583	203	#undef RSEQ_TEMPLATE_MO_RELEASE
abf9e855	204	#undef RSEQ_TEMPLATE_INDEX_CPU_ID
784b0012	205
a66de583	206	/* Per-mm-cid indexing. */
784b0012	207
abf9e855	208	#define RSEQ_TEMPLATE_INDEX_MM_CID
a66de583	209	#define RSEQ_TEMPLATE_MO_RELAXED
44ec21eb	210	#include "rseq/arch/arm/bits.h"
a66de583	211	#undef RSEQ_TEMPLATE_MO_RELAXED
784b0012	212
a66de583	213	#define RSEQ_TEMPLATE_MO_RELEASE
44ec21eb	214	#include "rseq/arch/arm/bits.h"
a66de583	215	#undef RSEQ_TEMPLATE_MO_RELEASE
abf9e855	216	#undef RSEQ_TEMPLATE_INDEX_MM_CID
784b0012	217
abf9e855	218	/* APIs which are not indexed. */
784b0012	219
abf9e855	220	#define RSEQ_TEMPLATE_INDEX_NONE
a66de583	221	#define RSEQ_TEMPLATE_MO_RELAXED
44ec21eb	222	#include "rseq/arch/arm/bits.h"
a66de583	223	#undef RSEQ_TEMPLATE_MO_RELAXED
abf9e855	224	#undef RSEQ_TEMPLATE_INDEX_NONE