[librseq.git] / include / rseq / arch / riscv.h

/* SPDX-License-Identifier: MIT */
/* SPDX-FileCopyrightText: 2022 Vincent Chen <vincent.chen@sifive.com> */
/* SPDX-FileCopyrightText: 2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */

/*
 * rseq-riscv.h
 */

/*
 * RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those
 * are not part of the public API.
 */

#ifndef _RSEQ_RSEQ_H
#error "Never use <rseq/arch/riscv.h> directly; include <rseq/rseq.h> instead."
#endif

/*
 * Select the instruction "csrw mhartid, x0" as the RSEQ_SIG. Unlike
 * other architectures, the ebreak instruction has no immediate field for
 * distinguishing purposes. Hence, ebreak is not suitable as RSEQ_SIG.
 * "csrw mhartid, x0" can also satisfy the RSEQ requirement because it
 * is an uncommon instruction and will raise an illegal instruction
 * exception when executed in all modes.
 */
#include <endian.h>

#if defined(__BYTE_ORDER) ? (__BYTE_ORDER == __LITTLE_ENDIAN) : defined(__LITTLE_ENDIAN)
#define RSEQ_SIG   0xf1401073  /* csrr mhartid, x0 */
#else
#error "Currently, RSEQ only supports Little-Endian version"
#endif

/*
 * Instruction selection between 32-bit/64-bit. Used internally in the
 * rseq headers.
 */
#if __riscv_xlen == 64
#define __RSEQ_ASM_REG_SEL(a, b)	a
#elif __riscv_xlen == 32
#define __RSEQ_ASM_REG_SEL(a, b)	b
#endif

#define RSEQ_ASM_REG_L	__RSEQ_ASM_REG_SEL("ld ", "lw ")
#define RSEQ_ASM_REG_S	__RSEQ_ASM_REG_SEL("sd ", "sw ")

/*
 * Refer to the Linux kernel memory model (LKMM) for documentation of
 * the memory barriers.
 */

/* Only used internally in rseq headers. */
#define RSEQ_ASM_RISCV_FENCE(p, s) \
	__asm__ __volatile__ ("fence " #p "," #s : : : "memory")
/* CPU memory barrier. */
#define rseq_smp_mb()	RSEQ_ASM_RISCV_FENCE(rw, rw)
/* CPU read memory barrier */
#define rseq_smp_rmb()	RSEQ_ASM_RISCV_FENCE(r, r)
/* CPU write memory barrier */
#define rseq_smp_wmb()	RSEQ_ASM_RISCV_FENCE(w, w)

/* Acquire: One-way permeable barrier. */
#define rseq_smp_load_acquire(p)					\
__extension__ ({							\
	rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p));	\
	RSEQ_ASM_RISCV_FENCE(r, rw);					\
	____p1;								\
})

/* Acquire barrier after control dependency. */
#define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_rmb()

/* Release: One-way permeable barrier. */
#define rseq_smp_store_release(p, v)					\
do {									\
	RSEQ_ASM_RISCV_FENCE(rw, w);					\
	RSEQ_WRITE_ONCE(*(p), v);					\
} while (0)

#define RSEQ_ASM_U64_PTR(x)	".quad " x
#define RSEQ_ASM_U32(x)		".long " x

/* Temporary registers. */
#define RSEQ_ASM_TMP_REG_1	"t6"
#define RSEQ_ASM_TMP_REG_2	"t5"
#define RSEQ_ASM_TMP_REG_3	"t4"
#define RSEQ_ASM_TMP_REG_4	"t3"

/* Only used in RSEQ_ASM_DEFINE_TABLE. */
#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip,	\
				post_commit_offset, abort_ip)		\
	".pushsection	__rseq_cs, \"aw\"\n"				\
	".balign	32\n"						\
	__rseq_str(label) ":\n"						\
	RSEQ_ASM_U32(__rseq_str(version)) "\n"				\
	RSEQ_ASM_U32(__rseq_str(flags)) "\n"				\
	RSEQ_ASM_U64_PTR(__rseq_str(start_ip)) "\n"			\
	RSEQ_ASM_U64_PTR(__rseq_str(post_commit_offset)) "\n"		\
	RSEQ_ASM_U64_PTR(__rseq_str(abort_ip)) "\n"			\
	".popsection\n\t"						\
	".pushsection __rseq_cs_ptr_array, \"aw\"\n"			\
	RSEQ_ASM_U64_PTR(__rseq_str(label) "b") "\n"			\
	".popsection\n"

/*
 * Define an rseq critical section structure of version 0 with no flags.
 *
 *  @label:
 *    Local label for the beginning of the critical section descriptor
 *    structure.
 *  @start_ip:
 *    Pointer to the first instruction of the sequence of consecutive assembly
 *    instructions.
 *  @post_commit_ip:
 *    Pointer to the instruction after the last instruction of the sequence of
 *    consecutive assembly instructions.
 *  @abort_ip:
 *    Pointer to the instruction where to move the execution flow in case of
 *    abort of the sequence of consecutive assembly instructions.
 */
#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,		 \
				(post_commit_ip) - (start_ip), abort_ip)

/*
 * Define the @exit_ip pointer as an exit point for the sequence of consecutive
 * assembly instructions at @start_ip.
 *
 *  @start_ip:
 *    Pointer to the first instruction of the sequence of consecutive assembly
 *    instructions.
 *  @exit_ip:
 *    Pointer to an exit point instruction.
 *
 * Exit points of a rseq critical section consist of all instructions outside
 * of the critical section where a critical section can either branch to or
 * reach through the normal course of its execution. The abort IP and the
 * post-commit IP are already part of the __rseq_cs section and should not be
 * explicitly defined as additional exit points. Knowing all exit points is
 * useful to assist debuggers stepping over the critical section.
 */
#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
	".pushsection __rseq_exit_point_array, \"aw\"\n"		\
	RSEQ_ASM_U64_PTR(__rseq_str(start_ip)) "\n"			\
	RSEQ_ASM_U64_PTR(__rseq_str(exit_ip)) "\n"			\
	".popsection\n"

/*
 * Define a critical section abort handler.
 *
 *  @label:
 *    Local label to the abort handler.
 *  @teardown:
 *    Sequence of instructions to run on abort.
 *  @abort_label:
 *    C label to jump to at the end of the sequence.
 */
#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)		\
	"j	222f\n"							\
	".balign	4\n"						\
	RSEQ_ASM_U32(__rseq_str(RSEQ_SIG)) "\n"				\
	__rseq_str(label) ":\n"						\
	teardown							\
	"j	%l[" __rseq_str(abort_label) "]\n"			\
	"222:\n"

/* Jump to local label @label when @cpu_id != @current_cpu_id. */
#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)		\
	RSEQ_INJECT_ASM(1)						\
	"la	" RSEQ_ASM_TMP_REG_1 ", " __rseq_str(cs_label) "\n"	\
	RSEQ_ASM_REG_S	RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(rseq_cs) "]\n" \
	__rseq_str(label) ":\n"

/* Store @value to address @var. */
#define RSEQ_ASM_OP_STORE(value, var)					\
	RSEQ_ASM_REG_S	"%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"

/* Jump to local label @label when @var != @expect. */
#define RSEQ_ASM_OP_CBNE(var, expect, label)				\
	RSEQ_ASM_REG_L	RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"	\
	"bne	" RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ,"	\
		  __rseq_str(label) "\n"

/*
 * Jump to local label @label when @var != @expect (32-bit register
 * comparison).
 */
#define RSEQ_ASM_OP_CBNE32(var, expect, label)				\
	"lw	" RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"	\
	"bne	" RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ,"	\
		  __rseq_str(label) "\n"

/* Jump to local label @label when @var == @expect. */
#define RSEQ_ASM_OP_CBEQ(var, expect, label)				\
	RSEQ_ASM_REG_L	RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"	\
	"beq	" RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ,"	\
		  __rseq_str(label) "\n"

/* Jump to local label @label when @cpu_id != @current_cpu_id. */
#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label)		\
	RSEQ_INJECT_ASM(2)						\
	RSEQ_ASM_OP_CBNE32(current_cpu_id, cpu_id, label)

/* Load @var into temporary register. */
#define RSEQ_ASM_OP_R_LOAD(var)						\
	RSEQ_ASM_REG_L	RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"

/* Store from temporary register into @var. */
#define RSEQ_ASM_OP_R_STORE(var)					\
	RSEQ_ASM_REG_S	RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"

/* Load from address in temporary register+@offset into temporary register. */
#define RSEQ_ASM_OP_R_LOAD_OFF(offset)					\
	"add	" RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(offset) "], "	\
		 RSEQ_ASM_TMP_REG_1 "\n"				\
	RSEQ_ASM_REG_L	RSEQ_ASM_TMP_REG_1 ", (" RSEQ_ASM_TMP_REG_1 ")\n"

/* Add @count to temporary register. */
#define RSEQ_ASM_OP_R_ADD(count)					\
	"add	" RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1		\
		", %[" __rseq_str(count) "]\n"

/*
 * End-of-sequence store of @value to address @var. Emit
 * @post_commit_label label after the store instruction.
 */
#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label)		\
	RSEQ_ASM_OP_STORE(value, var)					\
	__rseq_str(post_commit_label) ":\n"

/*
 * End-of-sequence store-release of @value to address @var. Emit
 * @post_commit_label label after the store instruction.
 */
#define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label)	\
	"fence	rw, w\n"						\
	RSEQ_ASM_OP_STORE(value, var)					\
	__rseq_str(post_commit_label) ":\n"

/*
 * End-of-sequence store of temporary register to address @var. Emit
 * @post_commit_label label after the store instruction.
 */
#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label)		\
	RSEQ_ASM_REG_S	RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"	\
	__rseq_str(post_commit_label) ":\n"

/*
 * Copy @len bytes from @src to @dst. This is an inefficient bytewise
 * copy and could be improved in the future.
 */
#define RSEQ_ASM_OP_R_BYTEWISE_MEMCPY(dst, src, len)			\
	"beqz	%[" __rseq_str(len) "], 333f\n"				\
	"mv	" RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(len) "]\n"	\
	"mv	" RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(src) "]\n"	\
	"mv	" RSEQ_ASM_TMP_REG_3 ", %[" __rseq_str(dst) "]\n"	\
	"222:\n"							\
	"lb	" RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_2 ")\n"	\
	"sb	" RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_3 ")\n"	\
	"addi	" RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 ", -1\n"	\
	"addi	" RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", 1\n"	\
	"addi	" RSEQ_ASM_TMP_REG_3 ", " RSEQ_ASM_TMP_REG_3 ", 1\n"	\
	"bnez	" RSEQ_ASM_TMP_REG_1 ", 222b\n"				\
	"333:\n"

/*
 * Load pointer address from @ptr. Add @off to offset from this pointer.
 * Add @inc to the resulting address as an end-of-sequence store.
 */
#define RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, inc, post_commit_label)	\
	"mv	" RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(ptr) "]\n"	\
	RSEQ_ASM_OP_R_ADD(off)						\
	RSEQ_ASM_REG_L	  RSEQ_ASM_TMP_REG_1 ", 0(" RSEQ_ASM_TMP_REG_1 ")\n" \
	RSEQ_ASM_OP_R_ADD(inc)						\
	__rseq_str(post_commit_label) ":\n"

/* Per-cpu-id indexing. */

#define RSEQ_TEMPLATE_INDEX_CPU_ID
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq/arch/riscv/bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED

#define RSEQ_TEMPLATE_MO_RELEASE
#include "rseq/arch/riscv/bits.h"
#undef RSEQ_TEMPLATE_MO_RELEASE
#undef RSEQ_TEMPLATE_INDEX_CPU_ID

/* Per-mm-cid indexing. */

#define RSEQ_TEMPLATE_INDEX_MM_CID
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq/arch/riscv/bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED

#define RSEQ_TEMPLATE_MO_RELEASE
#include "rseq/arch/riscv/bits.h"
#undef RSEQ_TEMPLATE_MO_RELEASE
#undef RSEQ_TEMPLATE_INDEX_MM_CID

/* APIs which are not indexed. */

#define RSEQ_TEMPLATE_INDEX_NONE
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq/arch/riscv/bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED
#undef RSEQ_TEMPLATE_INDEX_NONE
Commit	Line	Data
90702366	1	/* SPDX-License-Identifier: MIT */
f2d7b530	2	/* SPDX-FileCopyrightText: 2022 Vincent Chen <vincent.chen@sifive.com> */
c4f3965f MD	3	/* SPDX-FileCopyrightText: 2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
	4
	5	/*
	6	* rseq-riscv.h
	7	*/
	8
	9	/*
	10	* RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those
	11	* are not part of the public API.
	12	*/
f2d7b530	13
44ec21eb MJ	14	#ifndef _RSEQ_RSEQ_H
	15	#error "Never use <rseq/arch/riscv.h> directly; include <rseq/rseq.h> instead."
	16	#endif
	17
074b1077 MJ	18	/*
	19	* Select the instruction "csrw mhartid, x0" as the RSEQ_SIG. Unlike
	20	* other architectures, the ebreak instruction has no immediate field for
	21	* distinguishing purposes. Hence, ebreak is not suitable as RSEQ_SIG.
	22	* "csrw mhartid, x0" can also satisfy the RSEQ requirement because it
	23	* is an uncommon instruction and will raise an illegal instruction
	24	* exception when executed in all modes.
	25	*/
	26	#include <endian.h>
	27
	28	#if defined(__BYTE_ORDER) ? (__BYTE_ORDER == __LITTLE_ENDIAN) : defined(__LITTLE_ENDIAN)
	29	#define RSEQ_SIG 0xf1401073 /* csrr mhartid, x0 */
	30	#else
	31	#error "Currently, RSEQ only supports Little-Endian version"
	32	#endif
	33
c4f3965f MD	34	/*
	35	* Instruction selection between 32-bit/64-bit. Used internally in the
	36	* rseq headers.
	37	*/
074b1077	38	#if __riscv_xlen == 64
ad5902d4	39	#define __RSEQ_ASM_REG_SEL(a, b) a
074b1077	40	#elif __riscv_xlen == 32
ad5902d4	41	#define __RSEQ_ASM_REG_SEL(a, b) b
074b1077 MJ	42	#endif
074b1077 MJ	43
ad5902d4 MD	44	#define RSEQ_ASM_REG_L __RSEQ_ASM_REG_SEL("ld ", "lw ")
ad5902d4 MD	45	#define RSEQ_ASM_REG_S __RSEQ_ASM_REG_SEL("sd ", "sw ")
074b1077	46
c4f3965f MD	47	/*
	48	* Refer to the Linux kernel memory model (LKMM) for documentation of
	49	* the memory barriers.
	50	*/
	51
	52	/* Only used internally in rseq headers. */
ad5902d4	53	#define RSEQ_ASM_RISCV_FENCE(p, s) \
074b1077	54	__asm__ __volatile__ ("fence " #p "," #s : : : "memory")
c4f3965f	55	/* CPU memory barrier. */
ad5902d4	56	#define rseq_smp_mb() RSEQ_ASM_RISCV_FENCE(rw, rw)
c4f3965f	57	/* CPU read memory barrier */
ad5902d4	58	#define rseq_smp_rmb() RSEQ_ASM_RISCV_FENCE(r, r)
c4f3965f	59	/* CPU write memory barrier */
ad5902d4	60	#define rseq_smp_wmb() RSEQ_ASM_RISCV_FENCE(w, w)
074b1077	61
c4f3965f	62	/* Acquire: One-way permeable barrier. */
074b1077 MJ	63	#define rseq_smp_load_acquire(p) \
074b1077 MJ	64	__extension__ ({ \
3664a718	65	rseq_unqual_scalar_typeof((p)) ____p1 = RSEQ_READ_ONCE((p)); \
ad5902d4	66	RSEQ_ASM_RISCV_FENCE(r, rw); \
074b1077 MJ	67	____p1; \
	68	})
	69
c4f3965f	70	/* Acquire barrier after control dependency. */
074b1077 MJ	71	#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
074b1077 MJ	72
c4f3965f	73	/* Release: One-way permeable barrier. */
074b1077 MJ	74	#define rseq_smp_store_release(p, v) \
074b1077 MJ	75	do { \
ad5902d4	76	RSEQ_ASM_RISCV_FENCE(rw, w); \
826417f6	77	RSEQ_WRITE_ONCE(*(p), v); \
074b1077 MJ	78	} while (0)
074b1077 MJ	79
9dc22f84 MD	80	#define RSEQ_ASM_U64_PTR(x) ".quad " x
	81	#define RSEQ_ASM_U32(x) ".long " x
	82
c4f3965f MD	83	/* Temporary registers. */
	84	#define RSEQ_ASM_TMP_REG_1 "t6"
	85	#define RSEQ_ASM_TMP_REG_2 "t5"
	86	#define RSEQ_ASM_TMP_REG_3 "t4"
	87	#define RSEQ_ASM_TMP_REG_4 "t3"
	88
	89	/* Only used in RSEQ_ASM_DEFINE_TABLE. */
074b1077 MJ	90	#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
	91	post_commit_offset, abort_ip) \
	92	".pushsection __rseq_cs, \"aw\"\n" \
	93	".balign 32\n" \
	94	__rseq_str(label) ":\n" \
9dc22f84 MD	95	RSEQ_ASM_U32(__rseq_str(version)) "\n" \
	96	RSEQ_ASM_U32(__rseq_str(flags)) "\n" \
	97	RSEQ_ASM_U64_PTR(__rseq_str(start_ip)) "\n" \
	98	RSEQ_ASM_U64_PTR(__rseq_str(post_commit_offset)) "\n" \
	99	RSEQ_ASM_U64_PTR(__rseq_str(abort_ip)) "\n" \
074b1077 MJ	100	".popsection\n\t" \
074b1077 MJ	101	".pushsection __rseq_cs_ptr_array, \"aw\"\n" \
9dc22f84	102	RSEQ_ASM_U64_PTR(__rseq_str(label) "b") "\n" \
074b1077 MJ	103	".popsection\n"
074b1077 MJ	104
c4f3965f MD	105	/*
	106	* Define an rseq critical section structure of version 0 with no flags.
	107	*
	108	* @label:
	109	* Local label for the beginning of the critical section descriptor
	110	* structure.
	111	* @start_ip:
	112	* Pointer to the first instruction of the sequence of consecutive assembly
	113	* instructions.
	114	* @post_commit_ip:
	115	* Pointer to the instruction after the last instruction of the sequence of
	116	* consecutive assembly instructions.
	117	* @abort_ip:
	118	* Pointer to the instruction where to move the execution flow in case of
	119	* abort of the sequence of consecutive assembly instructions.
	120	*/
074b1077 MJ	121	#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
074b1077 MJ	122	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
d15481d2	123	(post_commit_ip) - (start_ip), abort_ip)
074b1077 MJ	124
074b1077 MJ	125	/*
c4f3965f MD	126	* Define the @exit_ip pointer as an exit point for the sequence of consecutive
	127	* assembly instructions at @start_ip.
	128	*
	129	* @start_ip:
	130	* Pointer to the first instruction of the sequence of consecutive assembly
	131	* instructions.
	132	* @exit_ip:
	133	* Pointer to an exit point instruction.
	134	*
074b1077 MJ	135	* Exit points of a rseq critical section consist of all instructions outside
	136	* of the critical section where a critical section can either branch to or
	137	* reach through the normal course of its execution. The abort IP and the
	138	* post-commit IP are already part of the __rseq_cs section and should not be
	139	* explicitly defined as additional exit points. Knowing all exit points is
	140	* useful to assist debuggers stepping over the critical section.
	141	*/
	142	#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
	143	".pushsection __rseq_exit_point_array, \"aw\"\n" \
9dc22f84 MD	144	RSEQ_ASM_U64_PTR(__rseq_str(start_ip)) "\n" \
9dc22f84 MD	145	RSEQ_ASM_U64_PTR(__rseq_str(exit_ip)) "\n" \
074b1077 MJ	146	".popsection\n"
074b1077 MJ	147
c4f3965f MD	148	/*
	149	* Define a critical section abort handler.
	150	*
	151	* @label:
	152	* Local label to the abort handler.
	153	* @teardown:
	154	* Sequence of instructions to run on abort.
	155	* @abort_label:
	156	* C label to jump to at the end of the sequence.
	157	*/
ebd27573	158	#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
074b1077 MJ	159	"j 222f\n" \
074b1077 MJ	160	".balign 4\n" \
9dc22f84	161	RSEQ_ASM_U32(__rseq_str(RSEQ_SIG)) "\n" \
074b1077	162	__rseq_str(label) ":\n" \
ebd27573	163	teardown \
074b1077 MJ	164	"j %l[" __rseq_str(abort_label) "]\n" \
	165	"222:\n"
	166
c4f3965f MD	167	/* Jump to local label @label when @cpu_id != @current_cpu_id. */
	168	#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
	169	RSEQ_INJECT_ASM(1) \
	170	"la " RSEQ_ASM_TMP_REG_1 ", " __rseq_str(cs_label) "\n" \
	171	RSEQ_ASM_REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(rseq_cs) "]\n" \
	172	__rseq_str(label) ":\n"
	173
	174	/* Store @value to address @var. */
074b1077	175	#define RSEQ_ASM_OP_STORE(value, var) \
ad5902d4	176	RSEQ_ASM_REG_S "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
074b1077	177
c4f3965f	178	/* Jump to local label @label when @var != @expect. */
769ec9a5	179	#define RSEQ_ASM_OP_CBNE(var, expect, label) \
ad5902d4	180	RSEQ_ASM_REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \
074b1077 MJ	181	"bne " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \
	182	__rseq_str(label) "\n"
	183
c4f3965f MD	184	/*
	185	* Jump to local label @label when @var != @expect (32-bit register
	186	* comparison).
	187	*/
769ec9a5	188	#define RSEQ_ASM_OP_CBNE32(var, expect, label) \
074b1077 MJ	189	"lw " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \
	190	"bne " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \
	191	__rseq_str(label) "\n"
	192
c4f3965f	193	/* Jump to local label @label when @var == @expect. */
769ec9a5	194	#define RSEQ_ASM_OP_CBEQ(var, expect, label) \
ad5902d4	195	RSEQ_ASM_REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \
074b1077 MJ	196	"beq " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \
	197	__rseq_str(label) "\n"
	198
c4f3965f	199	/* Jump to local label @label when @cpu_id != @current_cpu_id. */
769ec9a5	200	#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \
074b1077	201	RSEQ_INJECT_ASM(2) \
769ec9a5	202	RSEQ_ASM_OP_CBNE32(current_cpu_id, cpu_id, label)
074b1077	203
c4f3965f	204	/* Load @var into temporary register. */
074b1077	205	#define RSEQ_ASM_OP_R_LOAD(var) \
ad5902d4	206	RSEQ_ASM_REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"
074b1077	207
c4f3965f	208	/* Store from temporary register into @var. */
074b1077	209	#define RSEQ_ASM_OP_R_STORE(var) \
ad5902d4	210	RSEQ_ASM_REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"
074b1077	211
c4f3965f	212	/* Load from address in temporary register+@offset into temporary register. */
074b1077 MJ	213	#define RSEQ_ASM_OP_R_LOAD_OFF(offset) \
	214	"add " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(offset) "], " \
	215	RSEQ_ASM_TMP_REG_1 "\n" \
ad5902d4	216	RSEQ_ASM_REG_L RSEQ_ASM_TMP_REG_1 ", (" RSEQ_ASM_TMP_REG_1 ")\n"
074b1077	217
c4f3965f	218	/* Add @count to temporary register. */
074b1077 MJ	219	#define RSEQ_ASM_OP_R_ADD(count) \
	220	"add " RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 \
	221	", %[" __rseq_str(count) "]\n"
	222
c4f3965f MD	223	/*
	224	* End-of-sequence store of @value to address @var. Emit
	225	* @post_commit_label label after the store instruction.
	226	*/
074b1077 MJ	227	#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
	228	RSEQ_ASM_OP_STORE(value, var) \
	229	__rseq_str(post_commit_label) ":\n"
	230
c4f3965f MD	231	/*
	232	* End-of-sequence store-release of @value to address @var. Emit
	233	* @post_commit_label label after the store instruction.
	234	*/
074b1077 MJ	235	#define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \
	236	"fence rw, w\n" \
	237	RSEQ_ASM_OP_STORE(value, var) \
	238	__rseq_str(post_commit_label) ":\n"
	239
c4f3965f MD	240	/*
	241	* End-of-sequence store of temporary register to address @var. Emit
	242	* @post_commit_label label after the store instruction.
	243	*/
074b1077	244	#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
ad5902d4	245	RSEQ_ASM_REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \
074b1077 MJ	246	__rseq_str(post_commit_label) ":\n"
074b1077 MJ	247
c4f3965f MD	248	/*
	249	* Copy @len bytes from @src to @dst. This is an inefficient bytewise
	250	* copy and could be improved in the future.
	251	*/
28880e72	252	#define RSEQ_ASM_OP_R_BYTEWISE_MEMCPY(dst, src, len) \
074b1077 MJ	253	"beqz %[" __rseq_str(len) "], 333f\n" \
	254	"mv " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(len) "]\n" \
	255	"mv " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(src) "]\n" \
	256	"mv " RSEQ_ASM_TMP_REG_3 ", %[" __rseq_str(dst) "]\n" \
	257	"222:\n" \
	258	"lb " RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_2 ")\n" \
	259	"sb " RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_3 ")\n" \
	260	"addi " RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 ", -1\n" \
	261	"addi " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", 1\n" \
	262	"addi " RSEQ_ASM_TMP_REG_3 ", " RSEQ_ASM_TMP_REG_3 ", 1\n" \
	263	"bnez " RSEQ_ASM_TMP_REG_1 ", 222b\n" \
	264	"333:\n"
	265
c4f3965f MD	266	/*
	267	* Load pointer address from @ptr. Add @off to offset from this pointer.
	268	* Add @inc to the resulting address as an end-of-sequence store.
	269	*/
bbf6c31a	270	#define RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, inc, post_commit_label) \
074b1077 MJ	271	"mv " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(ptr) "]\n" \
074b1077 MJ	272	RSEQ_ASM_OP_R_ADD(off) \
ad5902d4	273	RSEQ_ASM_REG_L RSEQ_ASM_TMP_REG_1 ", 0(" RSEQ_ASM_TMP_REG_1 ")\n" \
074b1077 MJ	274	RSEQ_ASM_OP_R_ADD(inc) \
	275	__rseq_str(post_commit_label) ":\n"
	276
154b6bde	277	/* Per-cpu-id indexing. */
074b1077	278
abf9e855	279	#define RSEQ_TEMPLATE_INDEX_CPU_ID
154b6bde	280	#define RSEQ_TEMPLATE_MO_RELAXED
44ec21eb	281	#include "rseq/arch/riscv/bits.h"
154b6bde	282	#undef RSEQ_TEMPLATE_MO_RELAXED
074b1077	283
154b6bde	284	#define RSEQ_TEMPLATE_MO_RELEASE
44ec21eb	285	#include "rseq/arch/riscv/bits.h"
154b6bde	286	#undef RSEQ_TEMPLATE_MO_RELEASE
abf9e855	287	#undef RSEQ_TEMPLATE_INDEX_CPU_ID
074b1077	288
154b6bde	289	/* Per-mm-cid indexing. */
074b1077	290
abf9e855	291	#define RSEQ_TEMPLATE_INDEX_MM_CID
154b6bde	292	#define RSEQ_TEMPLATE_MO_RELAXED
44ec21eb	293	#include "rseq/arch/riscv/bits.h"
154b6bde MD	294	#undef RSEQ_TEMPLATE_MO_RELAXED
	295
	296	#define RSEQ_TEMPLATE_MO_RELEASE
44ec21eb	297	#include "rseq/arch/riscv/bits.h"
154b6bde	298	#undef RSEQ_TEMPLATE_MO_RELEASE
abf9e855	299	#undef RSEQ_TEMPLATE_INDEX_MM_CID
154b6bde	300
abf9e855	301	/* APIs which are not indexed. */
154b6bde	302
abf9e855	303	#define RSEQ_TEMPLATE_INDEX_NONE
154b6bde	304	#define RSEQ_TEMPLATE_MO_RELAXED
44ec21eb	305	#include "rseq/arch/riscv/bits.h"
154b6bde	306	#undef RSEQ_TEMPLATE_MO_RELAXED
abf9e855	307	#undef RSEQ_TEMPLATE_INDEX_NONE