[librseq.git] / include / rseq / rseq-s390.h

/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */

/*
 * RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the
 * access-register mode nor the linkage stack this instruction will always
 * cause a special-operation exception (the trap-enabled bit in the DUCT
 * is and will stay 0). The instruction pattern is
 *	b2 ff 0f ff	trap4   4095(%r0)
 */
#define RSEQ_SIG	0xB2FF0FFF

#define rseq_smp_mb()	__asm__ __volatile__ ("bcr 15,0" ::: "memory")
#define rseq_smp_rmb()	rseq_smp_mb()
#define rseq_smp_wmb()	rseq_smp_mb()

#define rseq_smp_load_acquire(p)					\
__extension__ ({							\
	__typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);			\
	rseq_barrier();							\
	____p1;								\
})

#define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_rmb()

#define rseq_smp_store_release(p, v)					\
do {									\
	rseq_barrier();							\
	RSEQ_WRITE_ONCE(*p, v);						\
} while (0)

#ifdef RSEQ_SKIP_FASTPATH
#include "rseq-skip.h"
#else /* !RSEQ_SKIP_FASTPATH */

#ifdef __s390x__

#define RSEQ_CS_PTR		rseq_cs.ptr

#define LONG_L			"lg"
#define LONG_S			"stg"
#define LONG_LT_R		"ltgr"
#define LONG_CMP		"cg"
#define LONG_CMP_R		"cgr"
#define LONG_ADDI		"aghi"
#define LONG_ADD_R		"agr"

#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,			\
				start_ip, post_commit_offset, abort_ip)	\
		".pushsection __rseq_cs, \"aw\"\n\t"			\
		".balign 32\n\t"					\
		__rseq_str(label) ":\n\t"				\
		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
		".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
		".popsection\n\t"					\
		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
		".quad " __rseq_str(label) "b\n\t"			\
		".popsection\n\t"

/*
 * Exit points of a rseq critical section consist of all instructions outside
 * of the critical section where a critical section can either branch to or
 * reach through the normal course of its execution. The abort IP and the
 * post-commit IP are already part of the __rseq_cs section and should not be
 * explicitly defined as additional exit points. Knowing all exit points is
 * useful to assist debuggers stepping over the critical section.
 */
#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
		".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
		".popsection\n\t"

#elif __s390__

#define RSEQ_CS_PTR		rseq_cs.ptr.ptr32

#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,			\
				start_ip, post_commit_offset, abort_ip)	\
		".pushsection __rseq_cs, \"aw\"\n\t"			\
		".balign 32\n\t"					\
		__rseq_str(label) ":\n\t"				\
		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
		".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
		".popsection\n\t"					\
		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
		".long 0x0, " __rseq_str(label) "b\n\t"			\
		".popsection\n\t"

/*
 * Exit points of a rseq critical section consist of all instructions outside
 * of the critical section where a critical section can either branch to or
 * reach through the normal course of its execution. The abort IP and the
 * post-commit IP are already part of the __rseq_cs section and should not be
 * explicitly defined as additional exit points. Knowing all exit points is
 * useful to assist debuggers stepping over the critical section.
 */
#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
		".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
		".popsection\n\t"

#define LONG_L			"l"
#define LONG_S			"st"
#define LONG_LT_R		"ltr"
#define LONG_CMP		"c"
#define LONG_CMP_R		"cr"
#define LONG_ADDI		"ahi"
#define LONG_ADD_R		"ar"

#endif

#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,		\
				(post_commit_ip - start_ip), abort_ip)

#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)		\
		RSEQ_INJECT_ASM(1)					\
		"larl %%r0, " __rseq_str(cs_label) "\n\t"		\
		LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t"		\
		__rseq_str(label) ":\n\t"

#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)		\
		RSEQ_INJECT_ASM(2)					\
		"c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
		"jnz " __rseq_str(label) "\n\t"

#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)		\
		".pushsection __rseq_failure, \"ax\"\n\t"		\
		".long " __rseq_str(RSEQ_SIG) "\n\t"			\
		__rseq_str(label) ":\n\t"				\
		teardown						\
		"jg %l[" __rseq_str(abort_label) "]\n\t"		\
		".popsection\n\t"

#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)		\
		".pushsection __rseq_failure, \"ax\"\n\t"		\
		__rseq_str(label) ":\n\t"				\
		teardown						\
		"jg %l[" __rseq_str(cmpfail_label) "]\n\t"		\
		".popsection\n\t"

static inline __attribute__((always_inline))
int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
#endif
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		LONG_CMP " %[expect], %[v]\n\t"
		"jnz %l[cmpfail]\n\t"
		RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		LONG_CMP " %[expect], %[v]\n\t"
		"jnz %l[error2]\n\t"
#endif
		/* final store */
		LONG_S " %[newv], %[v]\n\t"
		"2:\n\t"
		RSEQ_INJECT_ASM(5)
		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
		  [rseq_cs]		"m" (rseq_get_abi()->RSEQ_CS_PTR),
		  [v]			"m" (*v),
		  [expect]		"r" (expect),
		  [newv]		"r" (newv)
		  RSEQ_INJECT_INPUT
		: "memory", "cc", "r0"
		  RSEQ_INJECT_CLOBBER
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2
#endif
	);
	rseq_after_asm_goto();
	return 0;
abort:
	rseq_after_asm_goto();
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	rseq_after_asm_goto();
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_after_asm_goto();
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_after_asm_goto();
	rseq_bug("expected value comparison failed");
#endif
}

/*
 * Compare @v against @expectnot. When it does _not_ match, load @v
 * into @load, and store the content of *@v + voffp into @v.
 */
static inline __attribute__((always_inline))
int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
			       long voffp, intptr_t *load, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
#endif
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		LONG_L " %%r1, %[v]\n\t"
		LONG_CMP_R " %%r1, %[expectnot]\n\t"
		"je %l[cmpfail]\n\t"
		RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		LONG_L " %%r1, %[v]\n\t"
		LONG_CMP_R " %%r1, %[expectnot]\n\t"
		"je %l[error2]\n\t"
#endif
		LONG_S " %%r1, %[load]\n\t"
		LONG_ADD_R " %%r1, %[voffp]\n\t"
		LONG_L " %%r1, 0(%%r1)\n\t"
		/* final store */
		LONG_S " %%r1, %[v]\n\t"
		"2:\n\t"
		RSEQ_INJECT_ASM(5)
		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
		  [rseq_cs]		"m" (rseq_get_abi()->RSEQ_CS_PTR),
		  /* final store input */
		  [v]			"m" (*v),
		  [expectnot]		"r" (expectnot),
		  [voffp]		"r" (voffp),
		  [load]		"m" (*load)
		  RSEQ_INJECT_INPUT
		: "memory", "cc", "r0", "r1"
		  RSEQ_INJECT_CLOBBER
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2
#endif
	);
	rseq_after_asm_goto();
	return 0;
abort:
	rseq_after_asm_goto();
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	rseq_after_asm_goto();
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_after_asm_goto();
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_after_asm_goto();
	rseq_bug("expected value comparison failed");
#endif
}

static inline __attribute__((always_inline))
int rseq_addv(intptr_t *v, intptr_t count, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
#endif
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
#endif
		LONG_L " %%r0, %[v]\n\t"
		LONG_ADD_R " %%r0, %[count]\n\t"
		/* final store */
		LONG_S " %%r0, %[v]\n\t"
		"2:\n\t"
		RSEQ_INJECT_ASM(4)
		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
		  [rseq_cs]		"m" (rseq_get_abi()->RSEQ_CS_PTR),
		  /* final store input */
		  [v]			"m" (*v),
		  [count]		"r" (count)
		  RSEQ_INJECT_INPUT
		: "memory", "cc", "r0"
		  RSEQ_INJECT_CLOBBER
		: abort
#ifdef RSEQ_COMPARE_TWICE
		  , error1
#endif
	);
	rseq_after_asm_goto();
	return 0;
abort:
	rseq_after_asm_goto();
	RSEQ_INJECT_FAILED
	return -1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_after_asm_goto();
	rseq_bug("cpu_id comparison failed");
#endif
}

static inline __attribute__((always_inline))
int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
				 intptr_t *v2, intptr_t newv2,
				 intptr_t newv, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
#endif
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		LONG_CMP " %[expect], %[v]\n\t"
		"jnz %l[cmpfail]\n\t"
		RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		LONG_CMP " %[expect], %[v]\n\t"
		"jnz %l[error2]\n\t"
#endif
		/* try store */
		LONG_S " %[newv2], %[v2]\n\t"
		RSEQ_INJECT_ASM(5)
		/* final store */
		LONG_S " %[newv], %[v]\n\t"
		"2:\n\t"
		RSEQ_INJECT_ASM(6)
		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
		  [rseq_cs]		"m" (rseq_get_abi()->RSEQ_CS_PTR),
		  /* try store input */
		  [v2]			"m" (*v2),
		  [newv2]		"r" (newv2),
		  /* final store input */
		  [v]			"m" (*v),
		  [expect]		"r" (expect),
		  [newv]		"r" (newv)
		  RSEQ_INJECT_INPUT
		: "memory", "cc", "r0"
		  RSEQ_INJECT_CLOBBER
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2
#endif
	);
	rseq_after_asm_goto();
	return 0;
abort:
	rseq_after_asm_goto();
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	rseq_after_asm_goto();
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_after_asm_goto();
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_after_asm_goto();
	rseq_bug("expected value comparison failed");
#endif
}

/* s390 is TSO. */
static inline __attribute__((always_inline))
int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
					 intptr_t *v2, intptr_t newv2,
					 intptr_t newv, int cpu)
{
	return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
}

static inline __attribute__((always_inline))
int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
			      intptr_t *v2, intptr_t expect2,
			      intptr_t newv, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
#endif
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		LONG_CMP " %[expect], %[v]\n\t"
		"jnz %l[cmpfail]\n\t"
		RSEQ_INJECT_ASM(4)
		LONG_CMP " %[expect2], %[v2]\n\t"
		"jnz %l[cmpfail]\n\t"
		RSEQ_INJECT_ASM(5)
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		LONG_CMP " %[expect], %[v]\n\t"
		"jnz %l[error2]\n\t"
		LONG_CMP " %[expect2], %[v2]\n\t"
		"jnz %l[error3]\n\t"
#endif
		/* final store */
		LONG_S " %[newv], %[v]\n\t"
		"2:\n\t"
		RSEQ_INJECT_ASM(6)
		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
		  [rseq_cs]		"m" (rseq_get_abi()->RSEQ_CS_PTR),
		  /* cmp2 input */
		  [v2]			"m" (*v2),
		  [expect2]		"r" (expect2),
		  /* final store input */
		  [v]			"m" (*v),
		  [expect]		"r" (expect),
		  [newv]		"r" (newv)
		  RSEQ_INJECT_INPUT
		: "memory", "cc", "r0"
		  RSEQ_INJECT_CLOBBER
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2, error3
#endif
	);
	rseq_after_asm_goto();
	return 0;
abort:
	rseq_after_asm_goto();
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	rseq_after_asm_goto();
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_after_asm_goto();
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_after_asm_goto();
	rseq_bug("1st expected value comparison failed");
error3:
	rseq_after_asm_goto();
	rseq_bug("2nd expected value comparison failed");
#endif
}

static inline __attribute__((always_inline))
int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
				 void *dst, void *src, size_t len,
				 intptr_t newv, int cpu)
{
	uint64_t rseq_scratch[3];

	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
#endif
		LONG_S " %[src], %[rseq_scratch0]\n\t"
		LONG_S " %[dst], %[rseq_scratch1]\n\t"
		LONG_S " %[len], %[rseq_scratch2]\n\t"
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		LONG_CMP " %[expect], %[v]\n\t"
		"jnz 5f\n\t"
		RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
		LONG_CMP " %[expect], %[v]\n\t"
		"jnz 7f\n\t"
#endif
		/* try memcpy */
		LONG_LT_R " %[len], %[len]\n\t"
		"jz 333f\n\t"
		"222:\n\t"
		"ic %%r0,0(%[src])\n\t"
		"stc %%r0,0(%[dst])\n\t"
		LONG_ADDI " %[src], 1\n\t"
		LONG_ADDI " %[dst], 1\n\t"
		LONG_ADDI " %[len], -1\n\t"
		"jnz 222b\n\t"
		"333:\n\t"
		RSEQ_INJECT_ASM(5)
		/* final store */
		LONG_S " %[newv], %[v]\n\t"
		"2:\n\t"
		RSEQ_INJECT_ASM(6)
		/* teardown */
		LONG_L " %[len], %[rseq_scratch2]\n\t"
		LONG_L " %[dst], %[rseq_scratch1]\n\t"
		LONG_L " %[src], %[rseq_scratch0]\n\t"
		RSEQ_ASM_DEFINE_ABORT(4,
			LONG_L " %[len], %[rseq_scratch2]\n\t"
			LONG_L " %[dst], %[rseq_scratch1]\n\t"
			LONG_L " %[src], %[rseq_scratch0]\n\t",
			abort)
		RSEQ_ASM_DEFINE_CMPFAIL(5,
			LONG_L " %[len], %[rseq_scratch2]\n\t"
			LONG_L " %[dst], %[rseq_scratch1]\n\t"
			LONG_L " %[src], %[rseq_scratch0]\n\t",
			cmpfail)
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_CMPFAIL(6,
			LONG_L " %[len], %[rseq_scratch2]\n\t"
			LONG_L " %[dst], %[rseq_scratch1]\n\t"
			LONG_L " %[src], %[rseq_scratch0]\n\t",
			error1)
		RSEQ_ASM_DEFINE_CMPFAIL(7,
			LONG_L " %[len], %[rseq_scratch2]\n\t"
			LONG_L " %[dst], %[rseq_scratch1]\n\t"
			LONG_L " %[src], %[rseq_scratch0]\n\t",
			error2)
#endif
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (rseq_get_abi()->cpu_id),
		  [rseq_cs]		"m" (rseq_get_abi()->RSEQ_CS_PTR),
		  /* final store input */
		  [v]			"m" (*v),
		  [expect]		"r" (expect),
		  [newv]		"r" (newv),
		  /* try memcpy input */
		  [dst]			"r" (dst),
		  [src]			"r" (src),
		  [len]			"r" (len),
		  [rseq_scratch0]	"m" (rseq_scratch[0]),
		  [rseq_scratch1]	"m" (rseq_scratch[1]),
		  [rseq_scratch2]	"m" (rseq_scratch[2])
		  RSEQ_INJECT_INPUT
		: "memory", "cc", "r0"
		  RSEQ_INJECT_CLOBBER
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2
#endif
	);
	rseq_after_asm_goto();
	return 0;
abort:
	rseq_after_asm_goto();
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	rseq_after_asm_goto();
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_after_asm_goto();
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_after_asm_goto();
	rseq_bug("expected value comparison failed");
#endif
}

/* s390 is TSO. */
static inline __attribute__((always_inline))
int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
					 void *dst, void *src, size_t len,
					 intptr_t newv, int cpu)
{
	return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
					    newv, cpu);
}

#endif /* !RSEQ_SKIP_FASTPATH */
Commit	Line	Data
744d0b8b	1	/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
4969e3fa	2
9c15f6a7 MS	3	/*
	4	* RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the
	5	* access-register mode nor the linkage stack this instruction will always
	6	* cause a special-operation exception (the trap-enabled bit in the DUCT
	7	* is and will stay 0). The instruction pattern is
	8	* b2 ff 0f ff trap4 4095(%r0)
	9	*/
	10	#define RSEQ_SIG 0xB2FF0FFF
4969e3fa MD	11
	12	#define rseq_smp_mb() __asm__ __volatile__ ("bcr 15,0" ::: "memory")
	13	#define rseq_smp_rmb() rseq_smp_mb()
	14	#define rseq_smp_wmb() rseq_smp_mb()
	15
	16	#define rseq_smp_load_acquire(p) \
	17	__extension__ ({ \
	18	__typeof(p) ____p1 = RSEQ_READ_ONCE(p); \
	19	rseq_barrier(); \
	20	____p1; \
	21	})
	22
	23	#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
	24
	25	#define rseq_smp_store_release(p, v) \
	26	do { \
	27	rseq_barrier(); \
	28	RSEQ_WRITE_ONCE(*p, v); \
	29	} while (0)
	30
	31	#ifdef RSEQ_SKIP_FASTPATH
	32	#include "rseq-skip.h"
	33	#else /* !RSEQ_SKIP_FASTPATH */
	34
	35	#ifdef __s390x__
	36
6f277965 MD	37	#define RSEQ_CS_PTR rseq_cs.ptr
6f277965 MD	38
4969e3fa MD	39	#define LONG_L "lg"
	40	#define LONG_S "stg"
	41	#define LONG_LT_R "ltgr"
	42	#define LONG_CMP "cg"
	43	#define LONG_CMP_R "cgr"
	44	#define LONG_ADDI "aghi"
	45	#define LONG_ADD_R "agr"
	46
	47	#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
	48	start_ip, post_commit_offset, abort_ip) \
dd01d0fb	49	".pushsection __rseq_cs, \"aw\"\n\t" \
4969e3fa MD	50	".balign 32\n\t" \
	51	__rseq_str(label) ":\n\t" \
	52	".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
	53	".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
dd01d0fb MD	54	".popsection\n\t" \
	55	".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
	56	".quad " __rseq_str(label) "b\n\t" \
4969e3fa MD	57	".popsection\n\t"
4969e3fa MD	58
90d9876e MD	59	/*
90d9876e MD	60	* Exit points of a rseq critical section consist of all instructions outside
fd622cad MD	61	* of the critical section where a critical section can either branch to or
	62	* reach through the normal course of its execution. The abort IP and the
	63	* post-commit IP are already part of the __rseq_cs section and should not be
	64	* explicitly defined as additional exit points. Knowing all exit points is
	65	* useful to assist debuggers stepping over the critical section.
90d9876e MD	66	*/
	67	#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
	68	".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
	69	".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
	70	".popsection\n\t"
	71
4969e3fa MD	72	#elif __s390__
4969e3fa MD	73
6f277965 MD	74	#define RSEQ_CS_PTR rseq_cs.ptr.ptr32
6f277965 MD	75
4969e3fa MD	76	#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
4969e3fa MD	77	start_ip, post_commit_offset, abort_ip) \
dd01d0fb	78	".pushsection __rseq_cs, \"aw\"\n\t" \
4969e3fa MD	79	".balign 32\n\t" \
	80	__rseq_str(label) ":\n\t" \
	81	".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
	82	".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
dd01d0fb MD	83	".popsection\n\t" \
	84	".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
	85	".long 0x0, " __rseq_str(label) "b\n\t" \
4969e3fa MD	86	".popsection\n\t"
4969e3fa MD	87
90d9876e MD	88	/*
90d9876e MD	89	* Exit points of a rseq critical section consist of all instructions outside
fd622cad MD	90	* of the critical section where a critical section can either branch to or
	91	* reach through the normal course of its execution. The abort IP and the
	92	* post-commit IP are already part of the __rseq_cs section and should not be
	93	* explicitly defined as additional exit points. Knowing all exit points is
	94	* useful to assist debuggers stepping over the critical section.
90d9876e MD	95	*/
	96	#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
	97	".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
	98	".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
	99	".popsection\n\t"
	100
4969e3fa MD	101	#define LONG_L "l"
	102	#define LONG_S "st"
	103	#define LONG_LT_R "ltr"
	104	#define LONG_CMP "c"
	105	#define LONG_CMP_R "cr"
	106	#define LONG_ADDI "ahi"
	107	#define LONG_ADD_R "ar"
	108
	109	#endif
	110
	111	#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
	112	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
	113	(post_commit_ip - start_ip), abort_ip)
	114
	115	#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
	116	RSEQ_INJECT_ASM(1) \
	117	"larl %%r0, " __rseq_str(cs_label) "\n\t" \
	118	LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t" \
	119	__rseq_str(label) ":\n\t"
	120
	121	#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
	122	RSEQ_INJECT_ASM(2) \
	123	"c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
	124	"jnz " __rseq_str(label) "\n\t"
	125
	126	#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
	127	".pushsection __rseq_failure, \"ax\"\n\t" \
	128	".long " __rseq_str(RSEQ_SIG) "\n\t" \
	129	__rseq_str(label) ":\n\t" \
	130	teardown \
90b9a2c8	131	"jg %l[" __rseq_str(abort_label) "]\n\t" \
4969e3fa MD	132	".popsection\n\t"
	133
	134	#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
	135	".pushsection __rseq_failure, \"ax\"\n\t" \
	136	__rseq_str(label) ":\n\t" \
	137	teardown \
90b9a2c8	138	"jg %l[" __rseq_str(cmpfail_label) "]\n\t" \
4969e3fa MD	139	".popsection\n\t"
	140
	141	static inline __attribute__((always_inline))
	142	int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
	143	{
	144	RSEQ_INJECT_C(9)
	145
	146	__asm__ __volatile__ goto (
	147	RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e MD	148	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
	149	#ifdef RSEQ_COMPARE_TWICE
	150	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
	151	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
	152	#endif
4969e3fa MD	153	/* Start rseq by storing table entry pointer into rseq_cs. */
	154	RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
	155	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
	156	RSEQ_INJECT_ASM(3)
	157	LONG_CMP " %[expect], %[v]\n\t"
	158	"jnz %l[cmpfail]\n\t"
	159	RSEQ_INJECT_ASM(4)
	160	#ifdef RSEQ_COMPARE_TWICE
	161	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
	162	LONG_CMP " %[expect], %[v]\n\t"
	163	"jnz %l[error2]\n\t"
	164	#endif
	165	/* final store */
	166	LONG_S " %[newv], %[v]\n\t"
	167	"2:\n\t"
	168	RSEQ_INJECT_ASM(5)
	169	RSEQ_ASM_DEFINE_ABORT(4, "", abort)
	170	: /* gcc asm goto does not allow outputs */
	171	: [cpu_id] "r" (cpu),
9698c399	172	[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
6f277965	173	[rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR),
4969e3fa MD	174	[v] "m" (*v),
	175	[expect] "r" (expect),
	176	[newv] "r" (newv)
	177	RSEQ_INJECT_INPUT
	178	: "memory", "cc", "r0"
	179	RSEQ_INJECT_CLOBBER
	180	: abort, cmpfail
	181	#ifdef RSEQ_COMPARE_TWICE
	182	, error1, error2
	183	#endif
	184	);
dd76f2d6	185	rseq_after_asm_goto();
4969e3fa MD	186	return 0;
4969e3fa MD	187	abort:
dd76f2d6	188	rseq_after_asm_goto();
4969e3fa MD	189	RSEQ_INJECT_FAILED
	190	return -1;
	191	cmpfail:
dd76f2d6	192	rseq_after_asm_goto();
4969e3fa MD	193	return 1;
	194	#ifdef RSEQ_COMPARE_TWICE
	195	error1:
dd76f2d6	196	rseq_after_asm_goto();
4969e3fa MD	197	rseq_bug("cpu_id comparison failed");
4969e3fa MD	198	error2:
dd76f2d6	199	rseq_after_asm_goto();
4969e3fa MD	200	rseq_bug("expected value comparison failed");
	201	#endif
	202	}
	203
	204	/*
	205	* Compare @v against @expectnot. When it does _not_ match, load @v
	206	* into @load, and store the content of *@v + voffp into @v.
	207	*/
	208	static inline __attribute__((always_inline))
	209	int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
d35eae6b	210	long voffp, intptr_t *load, int cpu)
4969e3fa MD	211	{
	212	RSEQ_INJECT_C(9)
	213
	214	__asm__ __volatile__ goto (
	215	RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e MD	216	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
	217	#ifdef RSEQ_COMPARE_TWICE
	218	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
	219	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
	220	#endif
4969e3fa MD	221	/* Start rseq by storing table entry pointer into rseq_cs. */
	222	RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
	223	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
	224	RSEQ_INJECT_ASM(3)
	225	LONG_L " %%r1, %[v]\n\t"
	226	LONG_CMP_R " %%r1, %[expectnot]\n\t"
	227	"je %l[cmpfail]\n\t"
	228	RSEQ_INJECT_ASM(4)
	229	#ifdef RSEQ_COMPARE_TWICE
	230	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
	231	LONG_L " %%r1, %[v]\n\t"
	232	LONG_CMP_R " %%r1, %[expectnot]\n\t"
	233	"je %l[error2]\n\t"
	234	#endif
	235	LONG_S " %%r1, %[load]\n\t"
	236	LONG_ADD_R " %%r1, %[voffp]\n\t"
	237	LONG_L " %%r1, 0(%%r1)\n\t"
	238	/* final store */
	239	LONG_S " %%r1, %[v]\n\t"
	240	"2:\n\t"
	241	RSEQ_INJECT_ASM(5)
	242	RSEQ_ASM_DEFINE_ABORT(4, "", abort)
	243	: /* gcc asm goto does not allow outputs */
	244	: [cpu_id] "r" (cpu),
9698c399	245	[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
6f277965	246	[rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR),
4969e3fa MD	247	/* final store input */
	248	[v] "m" (*v),
	249	[expectnot] "r" (expectnot),
	250	[voffp] "r" (voffp),
	251	[load] "m" (*load)
	252	RSEQ_INJECT_INPUT
	253	: "memory", "cc", "r0", "r1"
	254	RSEQ_INJECT_CLOBBER
	255	: abort, cmpfail
	256	#ifdef RSEQ_COMPARE_TWICE
	257	, error1, error2
	258	#endif
	259	);
dd76f2d6	260	rseq_after_asm_goto();
4969e3fa MD	261	return 0;
4969e3fa MD	262	abort:
dd76f2d6	263	rseq_after_asm_goto();
4969e3fa MD	264	RSEQ_INJECT_FAILED
	265	return -1;
	266	cmpfail:
dd76f2d6	267	rseq_after_asm_goto();
4969e3fa MD	268	return 1;
	269	#ifdef RSEQ_COMPARE_TWICE
	270	error1:
dd76f2d6	271	rseq_after_asm_goto();
4969e3fa MD	272	rseq_bug("cpu_id comparison failed");
4969e3fa MD	273	error2:
dd76f2d6	274	rseq_after_asm_goto();
4969e3fa MD	275	rseq_bug("expected value comparison failed");
	276	#endif
	277	}
	278
	279	static inline __attribute__((always_inline))
	280	int rseq_addv(intptr_t *v, intptr_t count, int cpu)
	281	{
	282	RSEQ_INJECT_C(9)
	283
	284	__asm__ __volatile__ goto (
	285	RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e MD	286	#ifdef RSEQ_COMPARE_TWICE
	287	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
	288	#endif
4969e3fa MD	289	/* Start rseq by storing table entry pointer into rseq_cs. */
	290	RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
	291	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
	292	RSEQ_INJECT_ASM(3)
	293	#ifdef RSEQ_COMPARE_TWICE
	294	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
	295	#endif
	296	LONG_L " %%r0, %[v]\n\t"
	297	LONG_ADD_R " %%r0, %[count]\n\t"
	298	/* final store */
	299	LONG_S " %%r0, %[v]\n\t"
	300	"2:\n\t"
	301	RSEQ_INJECT_ASM(4)
	302	RSEQ_ASM_DEFINE_ABORT(4, "", abort)
	303	: /* gcc asm goto does not allow outputs */
	304	: [cpu_id] "r" (cpu),
9698c399	305	[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
6f277965	306	[rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR),
4969e3fa MD	307	/* final store input */
	308	[v] "m" (*v),
	309	[count] "r" (count)
	310	RSEQ_INJECT_INPUT
	311	: "memory", "cc", "r0"
	312	RSEQ_INJECT_CLOBBER
	313	: abort
	314	#ifdef RSEQ_COMPARE_TWICE
	315	, error1
	316	#endif
	317	);
dd76f2d6	318	rseq_after_asm_goto();
4969e3fa MD	319	return 0;
4969e3fa MD	320	abort:
dd76f2d6	321	rseq_after_asm_goto();
4969e3fa MD	322	RSEQ_INJECT_FAILED
	323	return -1;
	324	#ifdef RSEQ_COMPARE_TWICE
	325	error1:
dd76f2d6	326	rseq_after_asm_goto();
4969e3fa MD	327	rseq_bug("cpu_id comparison failed");
	328	#endif
	329	}
	330
	331	static inline __attribute__((always_inline))
	332	int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
	333	intptr_t *v2, intptr_t newv2,
	334	intptr_t newv, int cpu)
	335	{
	336	RSEQ_INJECT_C(9)
	337
	338	__asm__ __volatile__ goto (
	339	RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e MD	340	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
	341	#ifdef RSEQ_COMPARE_TWICE
	342	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
	343	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
	344	#endif
4969e3fa MD	345	/* Start rseq by storing table entry pointer into rseq_cs. */
	346	RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
	347	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
	348	RSEQ_INJECT_ASM(3)
	349	LONG_CMP " %[expect], %[v]\n\t"
	350	"jnz %l[cmpfail]\n\t"
	351	RSEQ_INJECT_ASM(4)
	352	#ifdef RSEQ_COMPARE_TWICE
	353	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
	354	LONG_CMP " %[expect], %[v]\n\t"
	355	"jnz %l[error2]\n\t"
	356	#endif
	357	/* try store */
	358	LONG_S " %[newv2], %[v2]\n\t"
	359	RSEQ_INJECT_ASM(5)
	360	/* final store */
	361	LONG_S " %[newv], %[v]\n\t"
	362	"2:\n\t"
	363	RSEQ_INJECT_ASM(6)
	364	RSEQ_ASM_DEFINE_ABORT(4, "", abort)
	365	: /* gcc asm goto does not allow outputs */
	366	: [cpu_id] "r" (cpu),
9698c399	367	[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
6f277965	368	[rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR),
4969e3fa MD	369	/* try store input */
	370	[v2] "m" (*v2),
	371	[newv2] "r" (newv2),
	372	/* final store input */
	373	[v] "m" (*v),
	374	[expect] "r" (expect),
	375	[newv] "r" (newv)
	376	RSEQ_INJECT_INPUT
	377	: "memory", "cc", "r0"
	378	RSEQ_INJECT_CLOBBER
	379	: abort, cmpfail
	380	#ifdef RSEQ_COMPARE_TWICE
	381	, error1, error2
	382	#endif
	383	);
dd76f2d6	384	rseq_after_asm_goto();
4969e3fa MD	385	return 0;
4969e3fa MD	386	abort:
dd76f2d6	387	rseq_after_asm_goto();
4969e3fa MD	388	RSEQ_INJECT_FAILED
	389	return -1;
	390	cmpfail:
dd76f2d6	391	rseq_after_asm_goto();
4969e3fa MD	392	return 1;
	393	#ifdef RSEQ_COMPARE_TWICE
	394	error1:
dd76f2d6	395	rseq_after_asm_goto();
4969e3fa MD	396	rseq_bug("cpu_id comparison failed");
4969e3fa MD	397	error2:
dd76f2d6	398	rseq_after_asm_goto();
4969e3fa MD	399	rseq_bug("expected value comparison failed");
	400	#endif
	401	}
	402
	403	/* s390 is TSO. */
	404	static inline __attribute__((always_inline))
	405	int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
	406	intptr_t *v2, intptr_t newv2,
	407	intptr_t newv, int cpu)
	408	{
	409	return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
	410	}
	411
	412	static inline __attribute__((always_inline))
	413	int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
	414	intptr_t *v2, intptr_t expect2,
	415	intptr_t newv, int cpu)
	416	{
	417	RSEQ_INJECT_C(9)
	418
	419	__asm__ __volatile__ goto (
	420	RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e MD	421	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
	422	#ifdef RSEQ_COMPARE_TWICE
	423	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
	424	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
	425	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
	426	#endif
4969e3fa MD	427	/* Start rseq by storing table entry pointer into rseq_cs. */
	428	RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
	429	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
	430	RSEQ_INJECT_ASM(3)
	431	LONG_CMP " %[expect], %[v]\n\t"
	432	"jnz %l[cmpfail]\n\t"
	433	RSEQ_INJECT_ASM(4)
	434	LONG_CMP " %[expect2], %[v2]\n\t"
	435	"jnz %l[cmpfail]\n\t"
	436	RSEQ_INJECT_ASM(5)
	437	#ifdef RSEQ_COMPARE_TWICE
	438	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
	439	LONG_CMP " %[expect], %[v]\n\t"
	440	"jnz %l[error2]\n\t"
	441	LONG_CMP " %[expect2], %[v2]\n\t"
	442	"jnz %l[error3]\n\t"
	443	#endif
	444	/* final store */
	445	LONG_S " %[newv], %[v]\n\t"
	446	"2:\n\t"
	447	RSEQ_INJECT_ASM(6)
	448	RSEQ_ASM_DEFINE_ABORT(4, "", abort)
	449	: /* gcc asm goto does not allow outputs */
	450	: [cpu_id] "r" (cpu),
9698c399	451	[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
6f277965	452	[rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR),
4969e3fa MD	453	/* cmp2 input */
	454	[v2] "m" (*v2),
	455	[expect2] "r" (expect2),
	456	/* final store input */
	457	[v] "m" (*v),
	458	[expect] "r" (expect),
	459	[newv] "r" (newv)
	460	RSEQ_INJECT_INPUT
	461	: "memory", "cc", "r0"
	462	RSEQ_INJECT_CLOBBER
	463	: abort, cmpfail
	464	#ifdef RSEQ_COMPARE_TWICE
	465	, error1, error2, error3
	466	#endif
	467	);
dd76f2d6	468	rseq_after_asm_goto();
4969e3fa MD	469	return 0;
4969e3fa MD	470	abort:
dd76f2d6	471	rseq_after_asm_goto();
4969e3fa MD	472	RSEQ_INJECT_FAILED
	473	return -1;
	474	cmpfail:
dd76f2d6	475	rseq_after_asm_goto();
4969e3fa MD	476	return 1;
	477	#ifdef RSEQ_COMPARE_TWICE
	478	error1:
dd76f2d6	479	rseq_after_asm_goto();
4969e3fa MD	480	rseq_bug("cpu_id comparison failed");
4969e3fa MD	481	error2:
dd76f2d6	482	rseq_after_asm_goto();
4969e3fa MD	483	rseq_bug("1st expected value comparison failed");
4969e3fa MD	484	error3:
dd76f2d6	485	rseq_after_asm_goto();
4969e3fa MD	486	rseq_bug("2nd expected value comparison failed");
	487	#endif
	488	}
	489
	490	static inline __attribute__((always_inline))
	491	int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
	492	void dst, void src, size_t len,
	493	intptr_t newv, int cpu)
	494	{
	495	uint64_t rseq_scratch[3];
	496
	497	RSEQ_INJECT_C(9)
	498
	499	__asm__ __volatile__ goto (
	500	RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
90d9876e MD	501	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
	502	#ifdef RSEQ_COMPARE_TWICE
	503	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
	504	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
	505	#endif
4969e3fa MD	506	LONG_S " %[src], %[rseq_scratch0]\n\t"
	507	LONG_S " %[dst], %[rseq_scratch1]\n\t"
	508	LONG_S " %[len], %[rseq_scratch2]\n\t"
	509	/* Start rseq by storing table entry pointer into rseq_cs. */
	510	RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
	511	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
	512	RSEQ_INJECT_ASM(3)
	513	LONG_CMP " %[expect], %[v]\n\t"
	514	"jnz 5f\n\t"
	515	RSEQ_INJECT_ASM(4)
	516	#ifdef RSEQ_COMPARE_TWICE
	517	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
	518	LONG_CMP " %[expect], %[v]\n\t"
	519	"jnz 7f\n\t"
	520	#endif
	521	/* try memcpy */
	522	LONG_LT_R " %[len], %[len]\n\t"
	523	"jz 333f\n\t"
	524	"222:\n\t"
	525	"ic %%r0,0(%[src])\n\t"
	526	"stc %%r0,0(%[dst])\n\t"
	527	LONG_ADDI " %[src], 1\n\t"
	528	LONG_ADDI " %[dst], 1\n\t"
	529	LONG_ADDI " %[len], -1\n\t"
	530	"jnz 222b\n\t"
	531	"333:\n\t"
	532	RSEQ_INJECT_ASM(5)
	533	/* final store */
	534	LONG_S " %[newv], %[v]\n\t"
	535	"2:\n\t"
	536	RSEQ_INJECT_ASM(6)
	537	/* teardown */
	538	LONG_L " %[len], %[rseq_scratch2]\n\t"
	539	LONG_L " %[dst], %[rseq_scratch1]\n\t"
	540	LONG_L " %[src], %[rseq_scratch0]\n\t"
	541	RSEQ_ASM_DEFINE_ABORT(4,
	542	LONG_L " %[len], %[rseq_scratch2]\n\t"
	543	LONG_L " %[dst], %[rseq_scratch1]\n\t"
	544	LONG_L " %[src], %[rseq_scratch0]\n\t",
	545	abort)
	546	RSEQ_ASM_DEFINE_CMPFAIL(5,
	547	LONG_L " %[len], %[rseq_scratch2]\n\t"
	548	LONG_L " %[dst], %[rseq_scratch1]\n\t"
	549	LONG_L " %[src], %[rseq_scratch0]\n\t",
	550	cmpfail)
	551	#ifdef RSEQ_COMPARE_TWICE
	552	RSEQ_ASM_DEFINE_CMPFAIL(6,
	553	LONG_L " %[len], %[rseq_scratch2]\n\t"
	554	LONG_L " %[dst], %[rseq_scratch1]\n\t"
	555	LONG_L " %[src], %[rseq_scratch0]\n\t",
	556	error1)
	557	RSEQ_ASM_DEFINE_CMPFAIL(7,
	558	LONG_L " %[len], %[rseq_scratch2]\n\t"
	559	LONG_L " %[dst], %[rseq_scratch1]\n\t"
	560	LONG_L " %[src], %[rseq_scratch0]\n\t",
	561	error2)
	562	#endif
	563	: /* gcc asm goto does not allow outputs */
	564	: [cpu_id] "r" (cpu),
9698c399	565	[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
6f277965	566	[rseq_cs] "m" (rseq_get_abi()->RSEQ_CS_PTR),
4969e3fa MD	567	/* final store input */
	568	[v] "m" (*v),
	569	[expect] "r" (expect),
	570	[newv] "r" (newv),
	571	/* try memcpy input */
	572	[dst] "r" (dst),
	573	[src] "r" (src),
	574	[len] "r" (len),
	575	[rseq_scratch0] "m" (rseq_scratch[0]),
	576	[rseq_scratch1] "m" (rseq_scratch[1]),
	577	[rseq_scratch2] "m" (rseq_scratch[2])
	578	RSEQ_INJECT_INPUT
	579	: "memory", "cc", "r0"
	580	RSEQ_INJECT_CLOBBER
	581	: abort, cmpfail
	582	#ifdef RSEQ_COMPARE_TWICE
	583	, error1, error2
	584	#endif
	585	);
dd76f2d6	586	rseq_after_asm_goto();
4969e3fa MD	587	return 0;
4969e3fa MD	588	abort:
dd76f2d6	589	rseq_after_asm_goto();
4969e3fa MD	590	RSEQ_INJECT_FAILED
	591	return -1;
	592	cmpfail:
dd76f2d6	593	rseq_after_asm_goto();
4969e3fa MD	594	return 1;
	595	#ifdef RSEQ_COMPARE_TWICE
	596	error1:
dd76f2d6	597	rseq_after_asm_goto();
4969e3fa MD	598	rseq_bug("cpu_id comparison failed");
4969e3fa MD	599	error2:
dd76f2d6	600	rseq_after_asm_goto();
4969e3fa MD	601	rseq_bug("expected value comparison failed");
	602	#endif
	603	}
	604
	605	/* s390 is TSO. */
	606	static inline __attribute__((always_inline))
	607	int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
	608	void dst, void src, size_t len,
	609	intptr_t newv, int cpu)
	610	{
	611	return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
	612	newv, cpu);
	613	}
de28c254	614
4969e3fa	615	#endif /* !RSEQ_SKIP_FASTPATH */