[librseq.git] / include / rseq / rseq-riscv.h

/* SPDX-License-Identifier: MIT */
/* SPDX-FileCopyrightText: 2022 Vincent Chen <vincent.chen@sifive.com> */

/*
 * Select the instruction "csrw mhartid, x0" as the RSEQ_SIG. Unlike
 * other architectures, the ebreak instruction has no immediate field for
 * distinguishing purposes. Hence, ebreak is not suitable as RSEQ_SIG.
 * "csrw mhartid, x0" can also satisfy the RSEQ requirement because it
 * is an uncommon instruction and will raise an illegal instruction
 * exception when executed in all modes.
 */
#include <endian.h>

#if defined(__BYTE_ORDER) ? (__BYTE_ORDER == __LITTLE_ENDIAN) : defined(__LITTLE_ENDIAN)
#define RSEQ_SIG   0xf1401073  /* csrr mhartid, x0 */
#else
#error "Currently, RSEQ only supports Little-Endian version"
#endif

#if __riscv_xlen == 64
#define __REG_SEL(a, b)	a
#elif __riscv_xlen == 32
#define __REG_SEL(a, b)	b
#endif

#define REG_L	__REG_SEL("ld ", "lw ")
#define REG_S	__REG_SEL("sd ", "sw ")

#define RISCV_FENCE(p, s) \
	__asm__ __volatile__ ("fence " #p "," #s : : : "memory")
#define rseq_smp_mb()	RISCV_FENCE(rw, rw)
#define rseq_smp_rmb()	RISCV_FENCE(r, r)
#define rseq_smp_wmb()	RISCV_FENCE(w, w)
#define RSEQ_ASM_TMP_REG_1	"t6"
#define RSEQ_ASM_TMP_REG_2	"t5"
#define RSEQ_ASM_TMP_REG_3	"t4"
#define RSEQ_ASM_TMP_REG_4	"t3"

#define rseq_smp_load_acquire(p)					\
__extension__ ({							\
	rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p));	\
	RISCV_FENCE(r, rw);						\
	____p1;								\
})

#define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_rmb()

#define rseq_smp_store_release(p, v)					\
do {									\
	RISCV_FENCE(rw, w);						\
	RSEQ_WRITE_ONCE(*(p), v);					\
} while (0)

#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip,	\
				post_commit_offset, abort_ip)		\
	".pushsection	__rseq_cs, \"aw\"\n"				\
	".balign	32\n"						\
	__rseq_str(label) ":\n"						\
	".long	" __rseq_str(version) ", " __rseq_str(flags) "\n"	\
	".quad	" __rseq_str(start_ip) ", "				\
			  __rseq_str(post_commit_offset) ", "		\
			  __rseq_str(abort_ip) "\n"			\
	".popsection\n\t"						\
	".pushsection __rseq_cs_ptr_array, \"aw\"\n"			\
	".quad " __rseq_str(label) "b\n"				\
	".popsection\n"

#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,		 \
				((post_commit_ip) - (start_ip)), abort_ip)

/*
 * Exit points of a rseq critical section consist of all instructions outside
 * of the critical section where a critical section can either branch to or
 * reach through the normal course of its execution. The abort IP and the
 * post-commit IP are already part of the __rseq_cs section and should not be
 * explicitly defined as additional exit points. Knowing all exit points is
 * useful to assist debuggers stepping over the critical section.
 */
#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
	".pushsection __rseq_exit_point_array, \"aw\"\n"		\
	".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n"	\
	".popsection\n"

#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)		\
	RSEQ_INJECT_ASM(1)						\
	"la	" RSEQ_ASM_TMP_REG_1 ", " __rseq_str(cs_label) "\n"	\
	REG_S	RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(rseq_cs) "]\n"	\
	__rseq_str(label) ":\n"

#define RSEQ_ASM_DEFINE_ABORT(label, abort_label)			\
	"j	222f\n"							\
	".balign	4\n"						\
	".long "	__rseq_str(RSEQ_SIG) "\n"			\
	__rseq_str(label) ":\n"						\
	"j	%l[" __rseq_str(abort_label) "]\n"			\
	"222:\n"

#define RSEQ_ASM_OP_STORE(value, var)					\
	REG_S	"%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"

#define RSEQ_ASM_OP_CBNE(var, expect, label)				\
	REG_L	RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"		\
	"bne	" RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ,"	\
		  __rseq_str(label) "\n"

#define RSEQ_ASM_OP_CBNE32(var, expect, label)				\
	"lw	" RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"	\
	"bne	" RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ,"	\
		  __rseq_str(label) "\n"

#define RSEQ_ASM_OP_CBEQ(var, expect, label)				\
	REG_L	RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"		\
	"beq	" RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ,"	\
		  __rseq_str(label) "\n"

#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label)		\
	RSEQ_INJECT_ASM(2)						\
	RSEQ_ASM_OP_CBNE32(current_cpu_id, cpu_id, label)

#define RSEQ_ASM_OP_R_LOAD(var)						\
	REG_L	RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"

#define RSEQ_ASM_OP_R_STORE(var)					\
	REG_S	RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"

#define RSEQ_ASM_OP_R_LOAD_OFF(offset)					\
	"add	" RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(offset) "], "	\
		 RSEQ_ASM_TMP_REG_1 "\n"				\
	REG_L	RSEQ_ASM_TMP_REG_1 ", (" RSEQ_ASM_TMP_REG_1 ")\n"

#define RSEQ_ASM_OP_R_ADD(count)					\
	"add	" RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1		\
		", %[" __rseq_str(count) "]\n"

#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label)		\
	RSEQ_ASM_OP_STORE(value, var)					\
	__rseq_str(post_commit_label) ":\n"

#define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label)	\
	"fence	rw, w\n"						\
	RSEQ_ASM_OP_STORE(value, var)					\
	__rseq_str(post_commit_label) ":\n"

#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label)		\
	REG_S	RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"		\
	__rseq_str(post_commit_label) ":\n"

#define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)				\
	"beqz	%[" __rseq_str(len) "], 333f\n"				\
	"mv	" RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(len) "]\n"	\
	"mv	" RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(src) "]\n"	\
	"mv	" RSEQ_ASM_TMP_REG_3 ", %[" __rseq_str(dst) "]\n"	\
	"222:\n"							\
	"lb	" RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_2 ")\n"	\
	"sb	" RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_3 ")\n"	\
	"addi	" RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 ", -1\n"	\
	"addi	" RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", 1\n"	\
	"addi	" RSEQ_ASM_TMP_REG_3 ", " RSEQ_ASM_TMP_REG_3 ", 1\n"	\
	"bnez	" RSEQ_ASM_TMP_REG_1 ", 222b\n"				\
	"333:\n"

#define RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, post_commit_label)		\
	"mv	" RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(ptr) "]\n"	\
	RSEQ_ASM_OP_R_ADD(off)						\
	REG_L	  RSEQ_ASM_TMP_REG_1 ", 0(" RSEQ_ASM_TMP_REG_1 ")\n"	\
	RSEQ_ASM_OP_R_ADD(inc)						\
	__rseq_str(post_commit_label) ":\n"

/* Per-cpu-id indexing. */

#define RSEQ_TEMPLATE_CPU_ID
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq-riscv-bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED

#define RSEQ_TEMPLATE_MO_RELEASE
#include "rseq-riscv-bits.h"
#undef RSEQ_TEMPLATE_MO_RELEASE
#undef RSEQ_TEMPLATE_CPU_ID

/* Per-mm-cid indexing. */

#define RSEQ_TEMPLATE_MM_CID
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq-riscv-bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED

#define RSEQ_TEMPLATE_MO_RELEASE
#include "rseq-riscv-bits.h"
#undef RSEQ_TEMPLATE_MO_RELEASE
#undef RSEQ_TEMPLATE_MM_CID

/* APIs which are not based on cpu ids. */

#define RSEQ_TEMPLATE_CPU_ID_NONE
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq-riscv-bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED
#undef RSEQ_TEMPLATE_CPU_ID_NONE
Commit	Line	Data
90702366	1	/* SPDX-License-Identifier: MIT */
f2d7b530 MJ	2	/* SPDX-FileCopyrightText: 2022 Vincent Chen <vincent.chen@sifive.com> */
f2d7b530 MJ	3
074b1077 MJ	4	/*
	5	* Select the instruction "csrw mhartid, x0" as the RSEQ_SIG. Unlike
	6	* other architectures, the ebreak instruction has no immediate field for
	7	* distinguishing purposes. Hence, ebreak is not suitable as RSEQ_SIG.
	8	* "csrw mhartid, x0" can also satisfy the RSEQ requirement because it
	9	* is an uncommon instruction and will raise an illegal instruction
	10	* exception when executed in all modes.
	11	*/
	12	#include <endian.h>
	13
	14	#if defined(__BYTE_ORDER) ? (__BYTE_ORDER == __LITTLE_ENDIAN) : defined(__LITTLE_ENDIAN)
	15	#define RSEQ_SIG 0xf1401073 /* csrr mhartid, x0 */
	16	#else
	17	#error "Currently, RSEQ only supports Little-Endian version"
	18	#endif
	19
	20	#if __riscv_xlen == 64
	21	#define __REG_SEL(a, b) a
	22	#elif __riscv_xlen == 32
	23	#define __REG_SEL(a, b) b
	24	#endif
	25
	26	#define REG_L __REG_SEL("ld ", "lw ")
	27	#define REG_S __REG_SEL("sd ", "sw ")
	28
	29	#define RISCV_FENCE(p, s) \
	30	__asm__ __volatile__ ("fence " #p "," #s : : : "memory")
	31	#define rseq_smp_mb() RISCV_FENCE(rw, rw)
	32	#define rseq_smp_rmb() RISCV_FENCE(r, r)
	33	#define rseq_smp_wmb() RISCV_FENCE(w, w)
	34	#define RSEQ_ASM_TMP_REG_1 "t6"
	35	#define RSEQ_ASM_TMP_REG_2 "t5"
	36	#define RSEQ_ASM_TMP_REG_3 "t4"
	37	#define RSEQ_ASM_TMP_REG_4 "t3"
	38
	39	#define rseq_smp_load_acquire(p) \
	40	__extension__ ({ \
3664a718	41	rseq_unqual_scalar_typeof((p)) ____p1 = RSEQ_READ_ONCE((p)); \
074b1077 MJ	42	RISCV_FENCE(r, rw); \
	43	____p1; \
	44	})
	45
	46	#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
	47
	48	#define rseq_smp_store_release(p, v) \
	49	do { \
	50	RISCV_FENCE(rw, w); \
826417f6	51	RSEQ_WRITE_ONCE(*(p), v); \
074b1077 MJ	52	} while (0)
074b1077 MJ	53
074b1077 MJ	54	#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
	55	post_commit_offset, abort_ip) \
	56	".pushsection __rseq_cs, \"aw\"\n" \
	57	".balign 32\n" \
	58	__rseq_str(label) ":\n" \
	59	".long " __rseq_str(version) ", " __rseq_str(flags) "\n" \
	60	".quad " __rseq_str(start_ip) ", " \
	61	__rseq_str(post_commit_offset) ", " \
	62	__rseq_str(abort_ip) "\n" \
	63	".popsection\n\t" \
	64	".pushsection __rseq_cs_ptr_array, \"aw\"\n" \
	65	".quad " __rseq_str(label) "b\n" \
	66	".popsection\n"
	67
	68	#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
	69	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
	70	((post_commit_ip) - (start_ip)), abort_ip)
	71
	72	/*
	73	* Exit points of a rseq critical section consist of all instructions outside
	74	* of the critical section where a critical section can either branch to or
	75	* reach through the normal course of its execution. The abort IP and the
	76	* post-commit IP are already part of the __rseq_cs section and should not be
	77	* explicitly defined as additional exit points. Knowing all exit points is
	78	* useful to assist debuggers stepping over the critical section.
	79	*/
	80	#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
	81	".pushsection __rseq_exit_point_array, \"aw\"\n" \
	82	".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \
	83	".popsection\n"
	84
	85	#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
	86	RSEQ_INJECT_ASM(1) \
	87	"la " RSEQ_ASM_TMP_REG_1 ", " __rseq_str(cs_label) "\n" \
	88	REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(rseq_cs) "]\n" \
	89	__rseq_str(label) ":\n"
	90
	91	#define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
	92	"j 222f\n" \
	93	".balign 4\n" \
	94	".long " __rseq_str(RSEQ_SIG) "\n" \
	95	__rseq_str(label) ":\n" \
	96	"j %l[" __rseq_str(abort_label) "]\n" \
	97	"222:\n"
	98
	99	#define RSEQ_ASM_OP_STORE(value, var) \
	100	REG_S "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
	101
769ec9a5	102	#define RSEQ_ASM_OP_CBNE(var, expect, label) \
074b1077 MJ	103	REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \
	104	"bne " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \
	105	__rseq_str(label) "\n"
	106
769ec9a5	107	#define RSEQ_ASM_OP_CBNE32(var, expect, label) \
074b1077 MJ	108	"lw " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \
	109	"bne " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \
	110	__rseq_str(label) "\n"
	111
769ec9a5	112	#define RSEQ_ASM_OP_CBEQ(var, expect, label) \
074b1077 MJ	113	REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \
	114	"beq " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \
	115	__rseq_str(label) "\n"
	116
769ec9a5	117	#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \
074b1077	118	RSEQ_INJECT_ASM(2) \
769ec9a5	119	RSEQ_ASM_OP_CBNE32(current_cpu_id, cpu_id, label)
074b1077 MJ	120
	121	#define RSEQ_ASM_OP_R_LOAD(var) \
	122	REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"
	123
	124	#define RSEQ_ASM_OP_R_STORE(var) \
	125	REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"
	126
	127	#define RSEQ_ASM_OP_R_LOAD_OFF(offset) \
	128	"add " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(offset) "], " \
	129	RSEQ_ASM_TMP_REG_1 "\n" \
	130	REG_L RSEQ_ASM_TMP_REG_1 ", (" RSEQ_ASM_TMP_REG_1 ")\n"
	131
	132	#define RSEQ_ASM_OP_R_ADD(count) \
	133	"add " RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 \
	134	", %[" __rseq_str(count) "]\n"
	135
	136	#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
	137	RSEQ_ASM_OP_STORE(value, var) \
	138	__rseq_str(post_commit_label) ":\n"
	139
	140	#define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \
	141	"fence rw, w\n" \
	142	RSEQ_ASM_OP_STORE(value, var) \
	143	__rseq_str(post_commit_label) ":\n"
	144
	145	#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
	146	REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \
	147	__rseq_str(post_commit_label) ":\n"
	148
	149	#define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \
	150	"beqz %[" __rseq_str(len) "], 333f\n" \
	151	"mv " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(len) "]\n" \
	152	"mv " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(src) "]\n" \
	153	"mv " RSEQ_ASM_TMP_REG_3 ", %[" __rseq_str(dst) "]\n" \
	154	"222:\n" \
	155	"lb " RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_2 ")\n" \
	156	"sb " RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_3 ")\n" \
	157	"addi " RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 ", -1\n" \
	158	"addi " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", 1\n" \
	159	"addi " RSEQ_ASM_TMP_REG_3 ", " RSEQ_ASM_TMP_REG_3 ", 1\n" \
	160	"bnez " RSEQ_ASM_TMP_REG_1 ", 222b\n" \
	161	"333:\n"
	162
	163	#define RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, post_commit_label) \
	164	"mv " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(ptr) "]\n" \
	165	RSEQ_ASM_OP_R_ADD(off) \
	166	REG_L RSEQ_ASM_TMP_REG_1 ", 0(" RSEQ_ASM_TMP_REG_1 ")\n" \
	167	RSEQ_ASM_OP_R_ADD(inc) \
	168	__rseq_str(post_commit_label) ":\n"
	169
154b6bde	170	/* Per-cpu-id indexing. */
074b1077	171
154b6bde MD	172	#define RSEQ_TEMPLATE_CPU_ID
	173	#define RSEQ_TEMPLATE_MO_RELAXED
	174	#include "rseq-riscv-bits.h"
	175	#undef RSEQ_TEMPLATE_MO_RELAXED
074b1077	176
154b6bde MD	177	#define RSEQ_TEMPLATE_MO_RELEASE
	178	#include "rseq-riscv-bits.h"
	179	#undef RSEQ_TEMPLATE_MO_RELEASE
	180	#undef RSEQ_TEMPLATE_CPU_ID
074b1077	181
154b6bde	182	/* Per-mm-cid indexing. */
074b1077	183
154b6bde MD	184	#define RSEQ_TEMPLATE_MM_CID
	185	#define RSEQ_TEMPLATE_MO_RELAXED
	186	#include "rseq-riscv-bits.h"
	187	#undef RSEQ_TEMPLATE_MO_RELAXED
	188
	189	#define RSEQ_TEMPLATE_MO_RELEASE
	190	#include "rseq-riscv-bits.h"
	191	#undef RSEQ_TEMPLATE_MO_RELEASE
	192	#undef RSEQ_TEMPLATE_MM_CID
	193
	194	/* APIs which are not based on cpu ids. */
	195
	196	#define RSEQ_TEMPLATE_CPU_ID_NONE
	197	#define RSEQ_TEMPLATE_MO_RELAXED
	198	#include "rseq-riscv-bits.h"
	199	#undef RSEQ_TEMPLATE_MO_RELAXED
	200	#undef RSEQ_TEMPLATE_CPU_ID_NONE