[librseq.git] / include / rseq / rseq-arm64.h

/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
/*
 * rseq-arm64.h
 *
 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com>
 */

/*
 * aarch64 -mbig-endian generates mixed endianness code vs data:
 * little-endian code and big-endian data. Ensure the RSEQ_SIG signature
 * matches code endianness.
 */
#define RSEQ_SIG_CODE	0xd428bc00	/* BRK #0x45E0.  */

#ifdef __AARCH64EB__
#define RSEQ_SIG_DATA	0x00bc28d4	/* BRK #0x45E0.  */
#else
#define RSEQ_SIG_DATA	RSEQ_SIG_CODE
#endif

#define RSEQ_SIG	RSEQ_SIG_DATA

#define rseq_smp_mb()	__asm__ __volatile__ ("dmb ish" ::: "memory")
#define rseq_smp_rmb()	__asm__ __volatile__ ("dmb ishld" ::: "memory")
#define rseq_smp_wmb()	__asm__ __volatile__ ("dmb ishst" ::: "memory")

#define rseq_smp_load_acquire(p)						\
__extension__ ({								\
	__typeof(*p) ____p1;							\
	switch (sizeof(*p)) {							\
	case 1:									\
		asm volatile ("ldarb %w0, %1"					\
			: "=r" (*(__u8 *)p)					\
			: "Q" (*p) : "memory");					\
		break;								\
	case 2:									\
		asm volatile ("ldarh %w0, %1"					\
			: "=r" (*(__u16 *)p)					\
			: "Q" (*p) : "memory");					\
		break;								\
	case 4:									\
		asm volatile ("ldar %w0, %1"					\
			: "=r" (*(__u32 *)p)					\
			: "Q" (*p) : "memory");					\
		break;								\
	case 8:									\
		asm volatile ("ldar %0, %1"					\
			: "=r" (*(__u64 *)p)					\
			: "Q" (*p) : "memory");					\
		break;								\
	}									\
	____p1;									\
})

#define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_rmb()

#define rseq_smp_store_release(p, v)						\
do {										\
	switch (sizeof(*p)) {							\
	case 1:									\
		asm volatile ("stlrb %w1, %0"					\
				: "=Q" (*p)					\
				: "r" ((__u8)v)					\
				: "memory");					\
		break;								\
	case 2:									\
		asm volatile ("stlrh %w1, %0"					\
				: "=Q" (*p)					\
				: "r" ((__u16)v)				\
				: "memory");					\
		break;								\
	case 4:									\
		asm volatile ("stlr %w1, %0"					\
				: "=Q" (*p)					\
				: "r" ((__u32)v)				\
				: "memory");					\
		break;								\
	case 8:									\
		asm volatile ("stlr %1, %0"					\
				: "=Q" (*p)					\
				: "r" ((__u64)v)				\
				: "memory");					\
		break;								\
	}									\
} while (0)

#ifdef RSEQ_SKIP_FASTPATH
#include "rseq-skip.h"
#else /* !RSEQ_SKIP_FASTPATH */

#define RSEQ_ASM_TMP_REG32	"w15"
#define RSEQ_ASM_TMP_REG	"x15"
#define RSEQ_ASM_TMP_REG_2	"x14"

#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip,		\
				post_commit_offset, abort_ip)			\
	"	.pushsection	__rseq_cs, \"aw\"\n"				\
	"	.balign	32\n"							\
	__rseq_str(label) ":\n"							\
	"	.long	" __rseq_str(version) ", " __rseq_str(flags) "\n"	\
	"	.quad	" __rseq_str(start_ip) ", "				\
			  __rseq_str(post_commit_offset) ", "			\
			  __rseq_str(abort_ip) "\n"				\
	"	.popsection\n\t"						\
	"	.pushsection __rseq_cs_ptr_array, \"aw\"\n"				\
	"	.quad " __rseq_str(label) "b\n"					\
	"	.popsection\n"

#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip)	\
	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,			\
				(post_commit_ip - start_ip), abort_ip)

/*
 * Exit points of a rseq critical section consist of all instructions outside
 * of the critical section where a critical section can either branch to or
 * reach through the normal course of its execution. The abort IP and the
 * post-commit IP are already part of the __rseq_cs section and should not be
 * explicitly defined as additional exit points. Knowing all exit points is
 * useful to assist debuggers stepping over the critical section.
 */
#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)				\
	"	.pushsection __rseq_exit_point_array, \"aw\"\n"			\
	"	.quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n"	\
	"	.popsection\n"

#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)			\
	RSEQ_INJECT_ASM(1)							\
	"	adrp	" RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n"	\
	"	add	" RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG		\
			", :lo12:" __rseq_str(cs_label) "\n"			\
	"	str	" RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n"	\
	__rseq_str(label) ":\n"

#define RSEQ_ASM_DEFINE_ABORT(label, abort_label)				\
	"	b	222f\n"							\
	"	.inst 	"	__rseq_str(RSEQ_SIG_CODE) "\n"			\
	__rseq_str(label) ":\n"							\
	"	b	%l[" __rseq_str(abort_label) "]\n"			\
	"222:\n"

#define RSEQ_ASM_OP_STORE(value, var)						\
	"	str	%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"

#define RSEQ_ASM_OP_STORE_RELEASE(value, var)					\
	"	stlr	%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"

#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label)			\
	RSEQ_ASM_OP_STORE(value, var)						\
	__rseq_str(post_commit_label) ":\n"

#define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label)		\
	RSEQ_ASM_OP_STORE_RELEASE(value, var)					\
	__rseq_str(post_commit_label) ":\n"

#define RSEQ_ASM_OP_CMPEQ(var, expect, label)					\
	"	ldr	" RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"		\
	"	sub	" RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG		\
			", %[" __rseq_str(expect) "]\n"				\
	"	cbnz	" RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"

#define RSEQ_ASM_OP_CMPEQ32(var, expect, label)					\
	"	ldr	" RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n"	\
	"	sub	" RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32		\
			", %w[" __rseq_str(expect) "]\n"			\
	"	cbnz	" RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n"

#define RSEQ_ASM_OP_CMPNE(var, expect, label)					\
	"	ldr	" RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"		\
	"	sub	" RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG		\
			", %[" __rseq_str(expect) "]\n"				\
	"	cbz	" RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"

#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)			\
	RSEQ_INJECT_ASM(2)							\
	RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label)

#define RSEQ_ASM_OP_R_LOAD(var)							\
	"	ldr	" RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"

#define RSEQ_ASM_OP_R_STORE(var)						\
	"	str	" RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"

#define RSEQ_ASM_OP_R_LOAD_OFF(offset)						\
	"	ldr	" RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG		\
			", %[" __rseq_str(offset) "]]\n"

#define RSEQ_ASM_OP_R_ADD(count)						\
	"	add	" RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG		\
			", %[" __rseq_str(count) "]\n"

#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label)			\
	"	str	" RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"		\
	__rseq_str(post_commit_label) ":\n"

#define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)					\
	"	cbz	%[" __rseq_str(len) "], 333f\n"				\
	"	mov	" RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n"	\
	"222:	sub	" RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n"	\
	"	ldrb	" RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]"	\
			", " RSEQ_ASM_TMP_REG_2 "]\n"				\
	"	strb	" RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]"	\
			", " RSEQ_ASM_TMP_REG_2 "]\n"				\
	"	cbnz	" RSEQ_ASM_TMP_REG_2 ", 222b\n"				\
	"333:\n"

static inline __attribute__((always_inline))
int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
#endif
		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
		RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
#endif
		RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
		RSEQ_INJECT_ASM(5)
		RSEQ_ASM_DEFINE_ABORT(4, abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"Qo" (rseq_get_abi()->cpu_id),
		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs),
		  [v]			"Qo" (*v),
		  [expect]		"r" (expect),
		  [newv]		"r" (newv)
		  RSEQ_INJECT_INPUT
		: "memory", RSEQ_ASM_TMP_REG
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2
#endif
	);

	return 0;
abort:
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_bug("expected value comparison failed");
#endif
}

static inline __attribute__((always_inline))
int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
			       off_t voffp, intptr_t *load, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
#endif
		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
		RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
#endif
		RSEQ_ASM_OP_R_LOAD(v)
		RSEQ_ASM_OP_R_STORE(load)
		RSEQ_ASM_OP_R_LOAD_OFF(voffp)
		RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
		RSEQ_INJECT_ASM(5)
		RSEQ_ASM_DEFINE_ABORT(4, abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"Qo" (rseq_get_abi()->cpu_id),
		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs),
		  [v]			"Qo" (*v),
		  [expectnot]		"r" (expectnot),
		  [load]		"Qo" (*load),
		  [voffp]		"r" (voffp)
		  RSEQ_INJECT_INPUT
		: "memory", RSEQ_ASM_TMP_REG
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2
#endif
	);
	return 0;
abort:
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_bug("expected value comparison failed");
#endif
}

static inline __attribute__((always_inline))
int rseq_addv(intptr_t *v, intptr_t count, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
#endif
		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
#endif
		RSEQ_ASM_OP_R_LOAD(v)
		RSEQ_ASM_OP_R_ADD(count)
		RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
		RSEQ_INJECT_ASM(4)
		RSEQ_ASM_DEFINE_ABORT(4, abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"Qo" (rseq_get_abi()->cpu_id),
		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs),
		  [v]			"Qo" (*v),
		  [count]		"r" (count)
		  RSEQ_INJECT_INPUT
		: "memory", RSEQ_ASM_TMP_REG
		: abort
#ifdef RSEQ_COMPARE_TWICE
		  , error1
#endif
	);
	return 0;
abort:
	RSEQ_INJECT_FAILED
	return -1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_bug("cpu_id comparison failed");
#endif
}

static inline __attribute__((always_inline))
int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
				 intptr_t *v2, intptr_t newv2,
				 intptr_t newv, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
#endif
		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
		RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
#endif
		RSEQ_ASM_OP_STORE(newv2, v2)
		RSEQ_INJECT_ASM(5)
		RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
		RSEQ_INJECT_ASM(6)
		RSEQ_ASM_DEFINE_ABORT(4, abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"Qo" (rseq_get_abi()->cpu_id),
		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs),
		  [expect]		"r" (expect),
		  [v]			"Qo" (*v),
		  [newv]		"r" (newv),
		  [v2]			"Qo" (*v2),
		  [newv2]		"r" (newv2)
		  RSEQ_INJECT_INPUT
		: "memory", RSEQ_ASM_TMP_REG
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2
#endif
	);

	return 0;
abort:
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_bug("expected value comparison failed");
#endif
}

static inline __attribute__((always_inline))
int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
					 intptr_t *v2, intptr_t newv2,
					 intptr_t newv, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
#endif
		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
		RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
#endif
		RSEQ_ASM_OP_STORE(newv2, v2)
		RSEQ_INJECT_ASM(5)
		RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
		RSEQ_INJECT_ASM(6)
		RSEQ_ASM_DEFINE_ABORT(4, abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"Qo" (rseq_get_abi()->cpu_id),
		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs),
		  [expect]		"r" (expect),
		  [v]			"Qo" (*v),
		  [newv]		"r" (newv),
		  [v2]			"Qo" (*v2),
		  [newv2]		"r" (newv2)
		  RSEQ_INJECT_INPUT
		: "memory", RSEQ_ASM_TMP_REG
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2
#endif
	);

	return 0;
abort:
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_bug("expected value comparison failed");
#endif
}

static inline __attribute__((always_inline))
int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
			      intptr_t *v2, intptr_t expect2,
			      intptr_t newv, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3])
#endif
		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
		RSEQ_INJECT_ASM(4)
		RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
		RSEQ_INJECT_ASM(5)
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
		RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
#endif
		RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
		RSEQ_INJECT_ASM(6)
		RSEQ_ASM_DEFINE_ABORT(4, abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"Qo" (rseq_get_abi()->cpu_id),
		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs),
		  [v]			"Qo" (*v),
		  [expect]		"r" (expect),
		  [v2]			"Qo" (*v2),
		  [expect2]		"r" (expect2),
		  [newv]		"r" (newv)
		  RSEQ_INJECT_INPUT
		: "memory", RSEQ_ASM_TMP_REG
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2, error3
#endif
	);

	return 0;
abort:
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_bug("expected value comparison failed");
error3:
	rseq_bug("2nd expected value comparison failed");
#endif
}

static inline __attribute__((always_inline))
int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
				 void *dst, void *src, size_t len,
				 intptr_t newv, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
#endif
		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
		RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
#endif
		RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
		RSEQ_INJECT_ASM(5)
		RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
		RSEQ_INJECT_ASM(6)
		RSEQ_ASM_DEFINE_ABORT(4, abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"Qo" (rseq_get_abi()->cpu_id),
		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs),
		  [expect]		"r" (expect),
		  [v]			"Qo" (*v),
		  [newv]		"r" (newv),
		  [dst]			"r" (dst),
		  [src]			"r" (src),
		  [len]			"r" (len)
		  RSEQ_INJECT_INPUT
		: "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2
#endif
	);

	return 0;
abort:
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_bug("expected value comparison failed");
#endif
}

static inline __attribute__((always_inline))
int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
					 void *dst, void *src, size_t len,
					 intptr_t newv, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
		RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
#endif
		RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
		RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
#endif
		RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
		RSEQ_INJECT_ASM(5)
		RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
		RSEQ_INJECT_ASM(6)
		RSEQ_ASM_DEFINE_ABORT(4, abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"Qo" (rseq_get_abi()->cpu_id),
		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs),
		  [expect]		"r" (expect),
		  [v]			"Qo" (*v),
		  [newv]		"r" (newv),
		  [dst]			"r" (dst),
		  [src]			"r" (src),
		  [len]			"r" (len)
		  RSEQ_INJECT_INPUT
		: "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2
#endif
	);

	return 0;
abort:
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_bug("expected value comparison failed");
#endif
}

#endif /* !RSEQ_SKIP_FASTPATH */
Commit	Line	Data
67a741ac	1	/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
d78a16c2 MD	2	/*
	3	* rseq-arm64.h
	4	*
	5	* (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	6	* (C) Copyright 2018 - Will Deacon <will.deacon@arm.com>
	7	*/
	8
8b7c64f7 MD	9	/*
	10	* aarch64 -mbig-endian generates mixed endianness code vs data:
	11	* little-endian code and big-endian data. Ensure the RSEQ_SIG signature
	12	* matches code endianness.
	13	*/
	14	#define RSEQ_SIG_CODE 0xd428bc00 /* BRK #0x45E0. */
	15
	16	#ifdef __AARCH64EB__
	17	#define RSEQ_SIG_DATA 0x00bc28d4 /* BRK #0x45E0. */
	18	#else
	19	#define RSEQ_SIG_DATA RSEQ_SIG_CODE
	20	#endif
	21
	22	#define RSEQ_SIG RSEQ_SIG_DATA
d78a16c2 MD	23
	24	#define rseq_smp_mb() __asm__ __volatile__ ("dmb ish" ::: "memory")
	25	#define rseq_smp_rmb() __asm__ __volatile__ ("dmb ishld" ::: "memory")
	26	#define rseq_smp_wmb() __asm__ __volatile__ ("dmb ishst" ::: "memory")
	27
	28	#define rseq_smp_load_acquire(p) \
	29	__extension__ ({ \
	30	__typeof(*p) ____p1; \
	31	switch (sizeof(*p)) { \
	32	case 1: \
	33	asm volatile ("ldarb %w0, %1" \
	34	: "=r" ((__u8 )p) \
	35	: "Q" (*p) : "memory"); \
	36	break; \
	37	case 2: \
	38	asm volatile ("ldarh %w0, %1" \
	39	: "=r" ((__u16 )p) \
	40	: "Q" (*p) : "memory"); \
	41	break; \
	42	case 4: \
	43	asm volatile ("ldar %w0, %1" \
	44	: "=r" ((__u32 )p) \
	45	: "Q" (*p) : "memory"); \
	46	break; \
	47	case 8: \
	48	asm volatile ("ldar %0, %1" \
	49	: "=r" ((__u64 )p) \
	50	: "Q" (*p) : "memory"); \
	51	break; \
	52	} \
	53	____p1; \
	54	})
	55
	56	#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
	57
	58	#define rseq_smp_store_release(p, v) \
	59	do { \
	60	switch (sizeof(*p)) { \
	61	case 1: \
	62	asm volatile ("stlrb %w1, %0" \
	63	: "=Q" (*p) \
	64	: "r" ((__u8)v) \
	65	: "memory"); \
	66	break; \
	67	case 2: \
	68	asm volatile ("stlrh %w1, %0" \
	69	: "=Q" (*p) \
	70	: "r" ((__u16)v) \
	71	: "memory"); \
	72	break; \
	73	case 4: \
	74	asm volatile ("stlr %w1, %0" \
	75	: "=Q" (*p) \
	76	: "r" ((__u32)v) \
	77	: "memory"); \
	78	break; \
	79	case 8: \
	80	asm volatile ("stlr %1, %0" \
	81	: "=Q" (*p) \
	82	: "r" ((__u64)v) \
	83	: "memory"); \
	84	break; \
	85	} \
	86	} while (0)
87
88	#ifdef RSEQ_SKIP_FASTPATH
89	#include "rseq-skip.h"
90	#else /* !RSEQ_SKIP_FASTPATH */
91
92	#define RSEQ_ASM_TMP_REG32 "w15"
93	#define RSEQ_ASM_TMP_REG "x15"
94	#define RSEQ_ASM_TMP_REG_2 "x14"
95
96	#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
97	post_commit_offset, abort_ip) \
dd01d0fb	98	" .pushsection __rseq_cs, \"aw\"\n" \
d78a16c2 MD	99	" .balign 32\n" \
	100	__rseq_str(label) ":\n" \
	101	" .long " __rseq_str(version) ", " __rseq_str(flags) "\n" \
	102	" .quad " __rseq_str(start_ip) ", " \
	103	__rseq_str(post_commit_offset) ", " \
	104	__rseq_str(abort_ip) "\n" \
dd01d0fb MD	105	" .popsection\n\t" \
	106	" .pushsection __rseq_cs_ptr_array, \"aw\"\n" \
	107	" .quad " __rseq_str(label) "b\n" \
d78a16c2 MD	108	" .popsection\n"
	109
	110	#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
	111	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
	112	(post_commit_ip - start_ip), abort_ip)
	113
90d9876e MD	114	/*
90d9876e MD	115	* Exit points of a rseq critical section consist of all instructions outside
fd622cad MD	116	* of the critical section where a critical section can either branch to or
	117	* reach through the normal course of its execution. The abort IP and the
	118	* post-commit IP are already part of the __rseq_cs section and should not be
	119	* explicitly defined as additional exit points. Knowing all exit points is
	120	* useful to assist debuggers stepping over the critical section.
90d9876e MD	121	*/
	122	#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
	123	" .pushsection __rseq_exit_point_array, \"aw\"\n" \
	124	" .quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \
	125	" .popsection\n"
	126
d78a16c2 MD	127	#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
	128	RSEQ_INJECT_ASM(1) \
	129	" adrp " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n" \
	130	" add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
	131	", :lo12:" __rseq_str(cs_label) "\n" \
	132	" str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n" \
	133	__rseq_str(label) ":\n"
	134
	135	#define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
	136	" b 222f\n" \
8b7c64f7	137	" .inst " __rseq_str(RSEQ_SIG_CODE) "\n" \
d78a16c2 MD	138	__rseq_str(label) ":\n" \
	139	" b %l[" __rseq_str(abort_label) "]\n" \
	140	"222:\n"
	141
	142	#define RSEQ_ASM_OP_STORE(value, var) \
	143	" str %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
	144
	145	#define RSEQ_ASM_OP_STORE_RELEASE(value, var) \
	146	" stlr %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
	147
	148	#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
	149	RSEQ_ASM_OP_STORE(value, var) \
	150	__rseq_str(post_commit_label) ":\n"
	151
	152	#define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \
	153	RSEQ_ASM_OP_STORE_RELEASE(value, var) \
	154	__rseq_str(post_commit_label) ":\n"
	155
	156	#define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
	157	" ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
	158	" sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
	159	", %[" __rseq_str(expect) "]\n" \
	160	" cbnz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
	161
	162	#define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \
	163	" ldr " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n" \
	164	" sub " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32 \
	165	", %w[" __rseq_str(expect) "]\n" \
	166	" cbnz " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n"
	167
	168	#define RSEQ_ASM_OP_CMPNE(var, expect, label) \
	169	" ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
	170	" sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
	171	", %[" __rseq_str(expect) "]\n" \
	172	" cbz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
	173
	174	#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
	175	RSEQ_INJECT_ASM(2) \
	176	RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label)
	177
	178	#define RSEQ_ASM_OP_R_LOAD(var) \
	179	" ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
	180
	181	#define RSEQ_ASM_OP_R_STORE(var) \
	182	" str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
	183
	184	#define RSEQ_ASM_OP_R_LOAD_OFF(offset) \
	185	" ldr " RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG \
	186	", %[" __rseq_str(offset) "]]\n"
	187
	188	#define RSEQ_ASM_OP_R_ADD(count) \
	189	" add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
	190	", %[" __rseq_str(count) "]\n"
	191
	192	#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
	193	" str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
	194	__rseq_str(post_commit_label) ":\n"
	195
	196	#define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \
	197	" cbz %[" __rseq_str(len) "], 333f\n" \
	198	" mov " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n" \
	199	"222: sub " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n" \
	200	" ldrb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]" \
	201	", " RSEQ_ASM_TMP_REG_2 "]\n" \
202	" strb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]" \
203	", " RSEQ_ASM_TMP_REG_2 "]\n" \
204	" cbnz " RSEQ_ASM_TMP_REG_2 ", 222b\n" \
205	"333:\n"
206
207	static inline __attribute__((always_inline))
208	int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
209	{
210	RSEQ_INJECT_C(9)
211
212	__asm__ __volatile__ goto (
213	RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
90d9876e MD	214	RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
	215	#ifdef RSEQ_COMPARE_TWICE
	216	RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
	217	RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
	218	#endif
d78a16c2 MD	219	RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
	220	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
	221	RSEQ_INJECT_ASM(3)
	222	RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
	223	RSEQ_INJECT_ASM(4)
	224	#ifdef RSEQ_COMPARE_TWICE
	225	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
	226	RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
	227	#endif
	228	RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
	229	RSEQ_INJECT_ASM(5)
	230	RSEQ_ASM_DEFINE_ABORT(4, abort)
	231	: /* gcc asm goto does not allow outputs */
	232	: [cpu_id] "r" (cpu),
9698c399 MD	233	[current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
9698c399 MD	234	[rseq_cs] "m" (rseq_get_abi()->rseq_cs),
d78a16c2 MD	235	[v] "Qo" (*v),
	236	[expect] "r" (expect),
	237	[newv] "r" (newv)
	238	RSEQ_INJECT_INPUT
	239	: "memory", RSEQ_ASM_TMP_REG
	240	: abort, cmpfail
	241	#ifdef RSEQ_COMPARE_TWICE
	242	, error1, error2
	243	#endif
	244	);
	245
	246	return 0;
	247	abort:
	248	RSEQ_INJECT_FAILED
	249	return -1;
	250	cmpfail:
	251	return 1;
	252	#ifdef RSEQ_COMPARE_TWICE
	253	error1:
	254	rseq_bug("cpu_id comparison failed");
	255	error2:
	256	rseq_bug("expected value comparison failed");
	257	#endif
	258	}
	259
	260	static inline __attribute__((always_inline))
	261	int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
	262	off_t voffp, intptr_t *load, int cpu)
	263	{
	264	RSEQ_INJECT_C(9)
	265
	266	__asm__ __volatile__ goto (
	267	RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
90d9876e MD	268	RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
	269	#ifdef RSEQ_COMPARE_TWICE
	270	RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
	271	RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
	272	#endif
d78a16c2 MD	273	RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
	274	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
	275	RSEQ_INJECT_ASM(3)
	276	RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
	277	RSEQ_INJECT_ASM(4)
	278	#ifdef RSEQ_COMPARE_TWICE
	279	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
	280	RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
	281	#endif
	282	RSEQ_ASM_OP_R_LOAD(v)
	283	RSEQ_ASM_OP_R_STORE(load)
	284	RSEQ_ASM_OP_R_LOAD_OFF(voffp)
	285	RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
	286	RSEQ_INJECT_ASM(5)
	287	RSEQ_ASM_DEFINE_ABORT(4, abort)
	288	: /* gcc asm goto does not allow outputs */
	289	: [cpu_id] "r" (cpu),
9698c399 MD	290	[current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
9698c399 MD	291	[rseq_cs] "m" (rseq_get_abi()->rseq_cs),
d78a16c2 MD	292	[v] "Qo" (*v),
	293	[expectnot] "r" (expectnot),
	294	[load] "Qo" (*load),
	295	[voffp] "r" (voffp)
	296	RSEQ_INJECT_INPUT
	297	: "memory", RSEQ_ASM_TMP_REG
	298	: abort, cmpfail
	299	#ifdef RSEQ_COMPARE_TWICE
	300	, error1, error2
	301	#endif
	302	);
	303	return 0;
	304	abort:
	305	RSEQ_INJECT_FAILED
	306	return -1;
	307	cmpfail:
	308	return 1;
	309	#ifdef RSEQ_COMPARE_TWICE
	310	error1:
	311	rseq_bug("cpu_id comparison failed");
	312	error2:
	313	rseq_bug("expected value comparison failed");
	314	#endif
	315	}
	316
	317	static inline __attribute__((always_inline))
	318	int rseq_addv(intptr_t *v, intptr_t count, int cpu)
	319	{
	320	RSEQ_INJECT_C(9)
	321
	322	__asm__ __volatile__ goto (
	323	RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
90d9876e MD	324	#ifdef RSEQ_COMPARE_TWICE
	325	RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
	326	#endif
d78a16c2 MD	327	RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
	328	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
	329	RSEQ_INJECT_ASM(3)
	330	#ifdef RSEQ_COMPARE_TWICE
	331	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
	332	#endif
	333	RSEQ_ASM_OP_R_LOAD(v)
	334	RSEQ_ASM_OP_R_ADD(count)
	335	RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
	336	RSEQ_INJECT_ASM(4)
	337	RSEQ_ASM_DEFINE_ABORT(4, abort)
	338	: /* gcc asm goto does not allow outputs */
	339	: [cpu_id] "r" (cpu),
9698c399 MD	340	[current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
9698c399 MD	341	[rseq_cs] "m" (rseq_get_abi()->rseq_cs),
d78a16c2 MD	342	[v] "Qo" (*v),
	343	[count] "r" (count)
	344	RSEQ_INJECT_INPUT
	345	: "memory", RSEQ_ASM_TMP_REG
	346	: abort
	347	#ifdef RSEQ_COMPARE_TWICE
	348	, error1
	349	#endif
	350	);
	351	return 0;
	352	abort:
	353	RSEQ_INJECT_FAILED
	354	return -1;
	355	#ifdef RSEQ_COMPARE_TWICE
	356	error1:
	357	rseq_bug("cpu_id comparison failed");
	358	#endif
	359	}
	360
	361	static inline __attribute__((always_inline))
	362	int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
	363	intptr_t *v2, intptr_t newv2,
	364	intptr_t newv, int cpu)
	365	{
	366	RSEQ_INJECT_C(9)
	367
	368	__asm__ __volatile__ goto (
	369	RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
90d9876e MD	370	RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
	371	#ifdef RSEQ_COMPARE_TWICE
	372	RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
	373	RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
	374	#endif
d78a16c2 MD	375	RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
	376	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
	377	RSEQ_INJECT_ASM(3)
	378	RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
	379	RSEQ_INJECT_ASM(4)
	380	#ifdef RSEQ_COMPARE_TWICE
	381	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
	382	RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
	383	#endif
	384	RSEQ_ASM_OP_STORE(newv2, v2)
	385	RSEQ_INJECT_ASM(5)
	386	RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
	387	RSEQ_INJECT_ASM(6)
	388	RSEQ_ASM_DEFINE_ABORT(4, abort)
	389	: /* gcc asm goto does not allow outputs */
	390	: [cpu_id] "r" (cpu),
9698c399 MD	391	[current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
9698c399 MD	392	[rseq_cs] "m" (rseq_get_abi()->rseq_cs),
d78a16c2 MD	393	[expect] "r" (expect),
	394	[v] "Qo" (*v),
	395	[newv] "r" (newv),
	396	[v2] "Qo" (*v2),
	397	[newv2] "r" (newv2)
	398	RSEQ_INJECT_INPUT
	399	: "memory", RSEQ_ASM_TMP_REG
	400	: abort, cmpfail
	401	#ifdef RSEQ_COMPARE_TWICE
	402	, error1, error2
	403	#endif
	404	);
	405
	406	return 0;
	407	abort:
	408	RSEQ_INJECT_FAILED
	409	return -1;
	410	cmpfail:
	411	return 1;
	412	#ifdef RSEQ_COMPARE_TWICE
	413	error1:
	414	rseq_bug("cpu_id comparison failed");
	415	error2:
	416	rseq_bug("expected value comparison failed");
	417	#endif
	418	}
	419
	420	static inline __attribute__((always_inline))
	421	int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
	422	intptr_t *v2, intptr_t newv2,
	423	intptr_t newv, int cpu)
	424	{
	425	RSEQ_INJECT_C(9)
	426
	427	__asm__ __volatile__ goto (
	428	RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
90d9876e MD	429	RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
	430	#ifdef RSEQ_COMPARE_TWICE
	431	RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
	432	RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
	433	#endif
d78a16c2 MD	434	RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
	435	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
	436	RSEQ_INJECT_ASM(3)
	437	RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
	438	RSEQ_INJECT_ASM(4)
	439	#ifdef RSEQ_COMPARE_TWICE
	440	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
	441	RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
	442	#endif
	443	RSEQ_ASM_OP_STORE(newv2, v2)
	444	RSEQ_INJECT_ASM(5)
	445	RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
	446	RSEQ_INJECT_ASM(6)
	447	RSEQ_ASM_DEFINE_ABORT(4, abort)
	448	: /* gcc asm goto does not allow outputs */
	449	: [cpu_id] "r" (cpu),
9698c399 MD	450	[current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
9698c399 MD	451	[rseq_cs] "m" (rseq_get_abi()->rseq_cs),
d78a16c2 MD	452	[expect] "r" (expect),
	453	[v] "Qo" (*v),
	454	[newv] "r" (newv),
	455	[v2] "Qo" (*v2),
	456	[newv2] "r" (newv2)
	457	RSEQ_INJECT_INPUT
	458	: "memory", RSEQ_ASM_TMP_REG
	459	: abort, cmpfail
	460	#ifdef RSEQ_COMPARE_TWICE
	461	, error1, error2
	462	#endif
	463	);
	464
	465	return 0;
	466	abort:
	467	RSEQ_INJECT_FAILED
	468	return -1;
	469	cmpfail:
	470	return 1;
	471	#ifdef RSEQ_COMPARE_TWICE
	472	error1:
	473	rseq_bug("cpu_id comparison failed");
	474	error2:
	475	rseq_bug("expected value comparison failed");
	476	#endif
	477	}
	478
	479	static inline __attribute__((always_inline))
	480	int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
	481	intptr_t *v2, intptr_t expect2,
	482	intptr_t newv, int cpu)
	483	{
	484	RSEQ_INJECT_C(9)
	485
	486	__asm__ __volatile__ goto (
	487	RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
90d9876e MD	488	RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
	489	#ifdef RSEQ_COMPARE_TWICE
	490	RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
	491	RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
	492	RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3])
	493	#endif
d78a16c2 MD	494	RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
	495	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
	496	RSEQ_INJECT_ASM(3)
	497	RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
	498	RSEQ_INJECT_ASM(4)
	499	RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
	500	RSEQ_INJECT_ASM(5)
	501	#ifdef RSEQ_COMPARE_TWICE
	502	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
	503	RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
	504	RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
	505	#endif
	506	RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
	507	RSEQ_INJECT_ASM(6)
	508	RSEQ_ASM_DEFINE_ABORT(4, abort)
	509	: /* gcc asm goto does not allow outputs */
	510	: [cpu_id] "r" (cpu),
9698c399 MD	511	[current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
9698c399 MD	512	[rseq_cs] "m" (rseq_get_abi()->rseq_cs),
d78a16c2 MD	513	[v] "Qo" (*v),
	514	[expect] "r" (expect),
	515	[v2] "Qo" (*v2),
	516	[expect2] "r" (expect2),
	517	[newv] "r" (newv)
	518	RSEQ_INJECT_INPUT
	519	: "memory", RSEQ_ASM_TMP_REG
	520	: abort, cmpfail
	521	#ifdef RSEQ_COMPARE_TWICE
	522	, error1, error2, error3
	523	#endif
	524	);
	525
	526	return 0;
	527	abort:
	528	RSEQ_INJECT_FAILED
	529	return -1;
	530	cmpfail:
	531	return 1;
	532	#ifdef RSEQ_COMPARE_TWICE
	533	error1:
	534	rseq_bug("cpu_id comparison failed");
	535	error2:
	536	rseq_bug("expected value comparison failed");
	537	error3:
	538	rseq_bug("2nd expected value comparison failed");
	539	#endif
	540	}
	541
	542	static inline __attribute__((always_inline))
	543	int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
	544	void dst, void src, size_t len,
	545	intptr_t newv, int cpu)
	546	{
	547	RSEQ_INJECT_C(9)
	548
	549	__asm__ __volatile__ goto (
	550	RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
90d9876e MD	551	RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
	552	#ifdef RSEQ_COMPARE_TWICE
	553	RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
	554	RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
	555	#endif
d78a16c2 MD	556	RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
	557	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
	558	RSEQ_INJECT_ASM(3)
	559	RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
	560	RSEQ_INJECT_ASM(4)
	561	#ifdef RSEQ_COMPARE_TWICE
	562	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
	563	RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
	564	#endif
	565	RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
	566	RSEQ_INJECT_ASM(5)
	567	RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
	568	RSEQ_INJECT_ASM(6)
	569	RSEQ_ASM_DEFINE_ABORT(4, abort)
	570	: /* gcc asm goto does not allow outputs */
	571	: [cpu_id] "r" (cpu),
9698c399 MD	572	[current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
9698c399 MD	573	[rseq_cs] "m" (rseq_get_abi()->rseq_cs),
d78a16c2 MD	574	[expect] "r" (expect),
	575	[v] "Qo" (*v),
	576	[newv] "r" (newv),
	577	[dst] "r" (dst),
	578	[src] "r" (src),
	579	[len] "r" (len)
	580	RSEQ_INJECT_INPUT
	581	: "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
	582	: abort, cmpfail
	583	#ifdef RSEQ_COMPARE_TWICE
	584	, error1, error2
	585	#endif
	586	);
	587
	588	return 0;
	589	abort:
	590	RSEQ_INJECT_FAILED
	591	return -1;
	592	cmpfail:
	593	return 1;
	594	#ifdef RSEQ_COMPARE_TWICE
	595	error1:
	596	rseq_bug("cpu_id comparison failed");
	597	error2:
	598	rseq_bug("expected value comparison failed");
	599	#endif
	600	}
	601
	602	static inline __attribute__((always_inline))
	603	int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
	604	void dst, void src, size_t len,
	605	intptr_t newv, int cpu)
	606	{
	607	RSEQ_INJECT_C(9)
	608
	609	__asm__ __volatile__ goto (
	610	RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
90d9876e MD	611	RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
	612	#ifdef RSEQ_COMPARE_TWICE
	613	RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
	614	RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
	615	#endif
d78a16c2 MD	616	RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
	617	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
	618	RSEQ_INJECT_ASM(3)
	619	RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
	620	RSEQ_INJECT_ASM(4)
	621	#ifdef RSEQ_COMPARE_TWICE
	622	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
	623	RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
	624	#endif
	625	RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
	626	RSEQ_INJECT_ASM(5)
	627	RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
	628	RSEQ_INJECT_ASM(6)
	629	RSEQ_ASM_DEFINE_ABORT(4, abort)
	630	: /* gcc asm goto does not allow outputs */
	631	: [cpu_id] "r" (cpu),
9698c399 MD	632	[current_cpu_id] "Qo" (rseq_get_abi()->cpu_id),
9698c399 MD	633	[rseq_cs] "m" (rseq_get_abi()->rseq_cs),
d78a16c2 MD	634	[expect] "r" (expect),
	635	[v] "Qo" (*v),
	636	[newv] "r" (newv),
	637	[dst] "r" (dst),
	638	[src] "r" (src),
	639	[len] "r" (len)
	640	RSEQ_INJECT_INPUT
	641	: "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
	642	: abort, cmpfail
	643	#ifdef RSEQ_COMPARE_TWICE
	644	, error1, error2
	645	#endif
	646	);
	647
	648	return 0;
	649	abort:
	650	RSEQ_INJECT_FAILED
	651	return -1;
	652	cmpfail:
	653	return 1;
	654	#ifdef RSEQ_COMPARE_TWICE
	655	error1:
	656	rseq_bug("cpu_id comparison failed");
	657	error2:
	658	rseq_bug("expected value comparison failed");
	659	#endif
	660	}
	661
	662	#endif /* !RSEQ_SKIP_FASTPATH */