[librseq.git] / include / rseq / rseq-ppc.h

/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
/*
 * rseq-ppc.h
 *
 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
 * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>
 */

#define RSEQ_SIG	0x53053053

#define rseq_smp_mb()		__asm__ __volatile__ ("sync"	::: "memory", "cc")
#define rseq_smp_lwsync()	__asm__ __volatile__ ("lwsync"	::: "memory", "cc")
#define rseq_smp_rmb()		rseq_smp_lwsync()
#define rseq_smp_wmb()		rseq_smp_lwsync()

#define rseq_smp_load_acquire(p)					\
__extension__ ({							\
	__typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);			\
	rseq_smp_lwsync();						\
	____p1;								\
})

#define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_lwsync()

#define rseq_smp_store_release(p, v)					\
do {									\
	rseq_smp_lwsync();						\
	RSEQ_WRITE_ONCE(*p, v);						\
} while (0)

#ifdef RSEQ_SKIP_FASTPATH
#include "rseq-skip.h"
#else /* !RSEQ_SKIP_FASTPATH */

/*
 * The __rseq_table section can be used by debuggers to better handle
 * single-stepping through the restartable critical sections.
 */

#ifdef __PPC64__

#define STORE_WORD	"std "
#define LOAD_WORD	"ld "
#define LOADX_WORD	"ldx "
#define CMP_WORD	"cmpd "

#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,				\
			start_ip, post_commit_offset, abort_ip)			\
		".pushsection __rseq_table, \"aw\"\n\t"				\
		".balign 32\n\t"						\
		__rseq_str(label) ":\n\t"					\
		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t"	\
		".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
		".popsection\n\t"

#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)			\
		RSEQ_INJECT_ASM(1)						\
		"lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t"		\
		"ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t"	\
		"rldicr %%r17, %%r17, 32, 31\n\t"				\
		"oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t"	\
		"ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t"		\
		"std %%r17, %[" __rseq_str(rseq_cs) "]\n\t"			\
		__rseq_str(label) ":\n\t"

#else /* #ifdef __PPC64__ */

#define STORE_WORD	"stw "
#define LOAD_WORD	"lwz "
#define LOADX_WORD	"lwzx "
#define CMP_WORD	"cmpw "

#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,				\
			start_ip, post_commit_offset, abort_ip)			\
		".pushsection __rseq_table, \"aw\"\n\t"				\
		".balign 32\n\t"						\
		__rseq_str(label) ":\n\t"					\
		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t"	\
		/* 32-bit only supported on BE */				\
		".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
		".popsection\n\t"

#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)			\
		RSEQ_INJECT_ASM(1)						\
		"lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t"			\
		"addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t"		\
		"stw %%r17, %[" __rseq_str(rseq_cs) "]\n\t"			\
		__rseq_str(label) ":\n\t"

#endif /* #ifdef __PPC64__ */

#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip)	\
		__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,		\
					(post_commit_ip - start_ip), abort_ip)

#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)			\
		RSEQ_INJECT_ASM(2)						\
		"lwz %%r17, %[" __rseq_str(current_cpu_id) "]\n\t"		\
		"cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t"		\
		"bne- cr7, " __rseq_str(label) "\n\t"

#define RSEQ_ASM_DEFINE_ABORT(label, abort_label)				\
		".pushsection __rseq_failure, \"ax\"\n\t"			\
		".long " __rseq_str(RSEQ_SIG) "\n\t"				\
		__rseq_str(label) ":\n\t"					\
		"b %l[" __rseq_str(abort_label) "]\n\t"				\
		".popsection\n\t"

/*
 * RSEQ_ASM_OPs: asm operations for rseq
 * 	RSEQ_ASM_OP_R_*: has hard-code registers in it
 * 	RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7)
 */
#define RSEQ_ASM_OP_CMPEQ(var, expect, label)					\
		LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"			\
		CMP_WORD "cr7, %%r17, %[" __rseq_str(expect) "]\n\t"		\
		"bne- cr7, " __rseq_str(label) "\n\t"

#define RSEQ_ASM_OP_CMPNE(var, expectnot, label)				\
		LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"			\
		CMP_WORD "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t"		\
		"beq- cr7, " __rseq_str(label) "\n\t"

#define RSEQ_ASM_OP_STORE(value, var)						\
		STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"

/* Load @var to r17 */
#define RSEQ_ASM_OP_R_LOAD(var)							\
		LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"

/* Store r17 to @var */
#define RSEQ_ASM_OP_R_STORE(var)						\
		STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t"

/* Add @count to r17 */
#define RSEQ_ASM_OP_R_ADD(count)						\
		"add %%r17, %[" __rseq_str(count) "], %%r17\n\t"

/* Load (r17 + voffp) to r17 */
#define RSEQ_ASM_OP_R_LOADX(voffp)						\
		LOADX_WORD "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"

/* TODO: implement a faster memcpy. */
#define RSEQ_ASM_OP_R_MEMCPY() \
		"cmpdi %%r19, 0\n\t" \
		"beq 333f\n\t" \
		"addi %%r20, %%r20, -1\n\t" \
		"addi %%r21, %%r21, -1\n\t" \
		"222:\n\t" \
		"lbzu %%r18, 1(%%r20)\n\t" \
		"stbu %%r18, 1(%%r21)\n\t" \
		"addi %%r19, %%r19, -1\n\t" \
		"cmpdi %%r19, 0\n\t" \
		"bne 222b\n\t" \
		"333:\n\t" \

#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label)			\
		STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t"			\
		__rseq_str(post_commit_label) ":\n\t"

#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label)			\
		STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
		__rseq_str(post_commit_label) ":\n\t"

static inline __attribute__((always_inline))
int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
		/* cmp cpuid */
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		/* cmp @v equal to @expect */
		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
		RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
		/* cmp cpuid */
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		/* cmp @v equal to @expect */
		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
#endif
		/* final store */
		RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
		RSEQ_INJECT_ASM(5)
		RSEQ_ASM_DEFINE_ABORT(4, abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
		  [v]			"m" (*v),
		  [expect]		"r" (expect),
		  [newv]		"r" (newv)
		  RSEQ_INJECT_INPUT
		: "memory", "cc", "r17"
		  RSEQ_INJECT_CLOBBER
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2
#endif
	);
	return 0;
abort:
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_bug("expected value comparison failed");
#endif
}

static inline __attribute__((always_inline))
int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
			       off_t voffp, intptr_t *load, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
		/* cmp cpuid */
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		/* cmp @v not equal to @expectnot */
		RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
		RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
		/* cmp cpuid */
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		/* cmp @v not equal to @expectnot */
		RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
#endif
		/* load the value of @v */
		RSEQ_ASM_OP_R_LOAD(v)
		/* store it in @load */
		RSEQ_ASM_OP_R_STORE(load)
		/* dereference voffp(v) */
		RSEQ_ASM_OP_R_LOADX(voffp)
		/* final store the value at voffp(v) */
		RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
		RSEQ_INJECT_ASM(5)
		RSEQ_ASM_DEFINE_ABORT(4, abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
		  /* final store input */
		  [v]			"m" (*v),
		  [expectnot]		"r" (expectnot),
		  [voffp]		"b" (voffp),
		  [load]		"m" (*load)
		  RSEQ_INJECT_INPUT
		: "memory", "cc", "r17"
		  RSEQ_INJECT_CLOBBER
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2
#endif
	);
	return 0;
abort:
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_bug("expected value comparison failed");
#endif
}

static inline __attribute__((always_inline))
int rseq_addv(intptr_t *v, intptr_t count, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
		/* cmp cpuid */
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
#ifdef RSEQ_COMPARE_TWICE
		/* cmp cpuid */
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
#endif
		/* load the value of @v */
		RSEQ_ASM_OP_R_LOAD(v)
		/* add @count to it */
		RSEQ_ASM_OP_R_ADD(count)
		/* final store */
		RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
		RSEQ_INJECT_ASM(4)
		RSEQ_ASM_DEFINE_ABORT(4, abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
		  /* final store input */
		  [v]			"m" (*v),
		  [count]		"r" (count)
		  RSEQ_INJECT_INPUT
		: "memory", "cc", "r17"
		  RSEQ_INJECT_CLOBBER
		: abort
#ifdef RSEQ_COMPARE_TWICE
		  , error1
#endif
	);
	return 0;
abort:
	RSEQ_INJECT_FAILED
	return -1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_bug("cpu_id comparison failed");
#endif
}

static inline __attribute__((always_inline))
int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
				 intptr_t *v2, intptr_t newv2,
				 intptr_t newv, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
		/* cmp cpuid */
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		/* cmp @v equal to @expect */
		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
		RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
		/* cmp cpuid */
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		/* cmp @v equal to @expect */
		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
#endif
		/* try store */
		RSEQ_ASM_OP_STORE(newv2, v2)
		RSEQ_INJECT_ASM(5)
		/* final store */
		RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
		RSEQ_INJECT_ASM(6)
		RSEQ_ASM_DEFINE_ABORT(4, abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
		  /* try store input */
		  [v2]			"m" (*v2),
		  [newv2]		"r" (newv2),
		  /* final store input */
		  [v]			"m" (*v),
		  [expect]		"r" (expect),
		  [newv]		"r" (newv)
		  RSEQ_INJECT_INPUT
		: "memory", "cc", "r17"
		  RSEQ_INJECT_CLOBBER
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2
#endif
	);
	return 0;
abort:
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_bug("expected value comparison failed");
#endif
}

static inline __attribute__((always_inline))
int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
					 intptr_t *v2, intptr_t newv2,
					 intptr_t newv, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
		/* cmp cpuid */
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		/* cmp @v equal to @expect */
		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
		RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
		/* cmp cpuid */
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		/* cmp @v equal to @expect */
		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
#endif
		/* try store */
		RSEQ_ASM_OP_STORE(newv2, v2)
		RSEQ_INJECT_ASM(5)
		/* for 'release' */
		"lwsync\n\t"
		/* final store */
		RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
		RSEQ_INJECT_ASM(6)
		RSEQ_ASM_DEFINE_ABORT(4, abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
		  /* try store input */
		  [v2]			"m" (*v2),
		  [newv2]		"r" (newv2),
		  /* final store input */
		  [v]			"m" (*v),
		  [expect]		"r" (expect),
		  [newv]		"r" (newv)
		  RSEQ_INJECT_INPUT
		: "memory", "cc", "r17"
		  RSEQ_INJECT_CLOBBER
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2
#endif
	);
	return 0;
abort:
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_bug("expected value comparison failed");
#endif
}

static inline __attribute__((always_inline))
int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
			      intptr_t *v2, intptr_t expect2,
			      intptr_t newv, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
		/* cmp cpuid */
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		/* cmp @v equal to @expect */
		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
		RSEQ_INJECT_ASM(4)
		/* cmp @v2 equal to @expct2 */
		RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
		RSEQ_INJECT_ASM(5)
#ifdef RSEQ_COMPARE_TWICE
		/* cmp cpuid */
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		/* cmp @v equal to @expect */
		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
		/* cmp @v2 equal to @expct2 */
		RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
#endif
		/* final store */
		RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
		RSEQ_INJECT_ASM(6)
		RSEQ_ASM_DEFINE_ABORT(4, abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
		  /* cmp2 input */
		  [v2]			"m" (*v2),
		  [expect2]		"r" (expect2),
		  /* final store input */
		  [v]			"m" (*v),
		  [expect]		"r" (expect),
		  [newv]		"r" (newv)
		  RSEQ_INJECT_INPUT
		: "memory", "cc", "r17"
		  RSEQ_INJECT_CLOBBER
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2, error3
#endif
	);
	return 0;
abort:
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_bug("1st expected value comparison failed");
error3:
	rseq_bug("2nd expected value comparison failed");
#endif
}

static inline __attribute__((always_inline))
int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
				 void *dst, void *src, size_t len,
				 intptr_t newv, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
		/* setup for mempcy */
		"mr %%r19, %[len]\n\t"
		"mr %%r20, %[src]\n\t"
		"mr %%r21, %[dst]\n\t"
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
		/* cmp cpuid */
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		/* cmp @v equal to @expect */
		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
		RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
		/* cmp cpuid */
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		/* cmp @v equal to @expect */
		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
#endif
		/* try memcpy */
		RSEQ_ASM_OP_R_MEMCPY()
		RSEQ_INJECT_ASM(5)
		/* final store */
		RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
		RSEQ_INJECT_ASM(6)
		/* teardown */
		RSEQ_ASM_DEFINE_ABORT(4, abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
		  /* final store input */
		  [v]			"m" (*v),
		  [expect]		"r" (expect),
		  [newv]		"r" (newv),
		  /* try memcpy input */
		  [dst]			"r" (dst),
		  [src]			"r" (src),
		  [len]			"r" (len)
		  RSEQ_INJECT_INPUT
		: "memory", "cc", "r17", "r18", "r19", "r20", "r21"
		  RSEQ_INJECT_CLOBBER
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2
#endif
	);
	return 0;
abort:
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_bug("expected value comparison failed");
#endif
}

static inline __attribute__((always_inline))
int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
					 void *dst, void *src, size_t len,
					 intptr_t newv, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
		/* setup for mempcy */
		"mr %%r19, %[len]\n\t"
		"mr %%r20, %[src]\n\t"
		"mr %%r21, %[dst]\n\t"
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
		/* cmp cpuid */
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		/* cmp @v equal to @expect */
		RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
		RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
		/* cmp cpuid */
		RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		/* cmp @v equal to @expect */
		RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
#endif
		/* try memcpy */
		RSEQ_ASM_OP_R_MEMCPY()
		RSEQ_INJECT_ASM(5)
		/* for 'release' */
		"lwsync\n\t"
		/* final store */
		RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
		RSEQ_INJECT_ASM(6)
		/* teardown */
		RSEQ_ASM_DEFINE_ABORT(4, abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (__rseq_abi.cpu_id),
		  [rseq_cs]		"m" (__rseq_abi.rseq_cs),
		  /* final store input */
		  [v]			"m" (*v),
		  [expect]		"r" (expect),
		  [newv]		"r" (newv),
		  /* try memcpy input */
		  [dst]			"r" (dst),
		  [src]			"r" (src),
		  [len]			"r" (len)
		  RSEQ_INJECT_INPUT
		: "memory", "cc", "r17", "r18", "r19", "r20", "r21"
		  RSEQ_INJECT_CLOBBER
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2
#endif
	);
	return 0;
abort:
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_bug("expected value comparison failed");
#endif
}

#undef STORE_WORD
#undef LOAD_WORD
#undef LOADX_WORD
#undef CMP_WORD

#endif /* !RSEQ_SKIP_FASTPATH */
Commit	Line	Data
744d0b8b	1	/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
784b0012 MD	2	/*
	3	* rseq-ppc.h
	4	*
	5	* (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
	6	* (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>
	7	*/
	8
	9	#define RSEQ_SIG 0x53053053
	10
	11	#define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory", "cc")
	12	#define rseq_smp_lwsync() __asm__ __volatile__ ("lwsync" ::: "memory", "cc")
	13	#define rseq_smp_rmb() rseq_smp_lwsync()
	14	#define rseq_smp_wmb() rseq_smp_lwsync()
	15
	16	#define rseq_smp_load_acquire(p) \
	17	__extension__ ({ \
	18	__typeof(p) ____p1 = RSEQ_READ_ONCE(p); \
	19	rseq_smp_lwsync(); \
	20	____p1; \
	21	})
	22
	23	#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_lwsync()
	24
	25	#define rseq_smp_store_release(p, v) \
	26	do { \
	27	rseq_smp_lwsync(); \
	28	RSEQ_WRITE_ONCE(*p, v); \
	29	} while (0)
	30
	31	#ifdef RSEQ_SKIP_FASTPATH
	32	#include "rseq-skip.h"
	33	#else /* !RSEQ_SKIP_FASTPATH */
	34
	35	/*
	36	* The __rseq_table section can be used by debuggers to better handle
	37	* single-stepping through the restartable critical sections.
	38	*/
	39
	40	#ifdef __PPC64__
	41
	42	#define STORE_WORD "std "
	43	#define LOAD_WORD "ld "
	44	#define LOADX_WORD "ldx "
	45	#define CMP_WORD "cmpd "
	46
	47	#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
	48	start_ip, post_commit_offset, abort_ip) \
	49	".pushsection __rseq_table, \"aw\"\n\t" \
	50	".balign 32\n\t" \
	51	__rseq_str(label) ":\n\t" \
	52	".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
	53	".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
	54	".popsection\n\t"
	55
	56	#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
	57	RSEQ_INJECT_ASM(1) \
	58	"lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t" \
	59	"ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t" \
	60	"rldicr %%r17, %%r17, 32, 31\n\t" \
	61	"oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t" \
	62	"ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
	63	"std %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
	64	__rseq_str(label) ":\n\t"
	65
66	#else /* #ifdef __PPC64__ */
67
68	#define STORE_WORD "stw "
69	#define LOAD_WORD "lwz "
70	#define LOADX_WORD "lwzx "
71	#define CMP_WORD "cmpw "
72
73	#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
74	start_ip, post_commit_offset, abort_ip) \
75	".pushsection __rseq_table, \"aw\"\n\t" \
76	".balign 32\n\t" \
77	__rseq_str(label) ":\n\t" \
78	".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
79	/* 32-bit only supported on BE */ \
80	".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
81	".popsection\n\t"
82
83	#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
84	RSEQ_INJECT_ASM(1) \
85	"lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t" \
86	"addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
87	"stw %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
88	__rseq_str(label) ":\n\t"
89
90	#endif /* #ifdef __PPC64__ */
91
92	#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
93	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
94	(post_commit_ip - start_ip), abort_ip)
95
96	#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
97	RSEQ_INJECT_ASM(2) \
98	"lwz %%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \
99	"cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t" \
100	"bne- cr7, " __rseq_str(label) "\n\t"
101
102	#define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
103	".pushsection __rseq_failure, \"ax\"\n\t" \
104	".long " __rseq_str(RSEQ_SIG) "\n\t" \
105	__rseq_str(label) ":\n\t" \
106	"b %l[" __rseq_str(abort_label) "]\n\t" \
107	".popsection\n\t"
108
109	/*
110	* RSEQ_ASM_OPs: asm operations for rseq
111	* RSEQ_ASM_OP_R_*: has hard-code registers in it
112	* RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7)
113	*/
114	#define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
115	LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \
116	CMP_WORD "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \
117	"bne- cr7, " __rseq_str(label) "\n\t"
118
119	#define RSEQ_ASM_OP_CMPNE(var, expectnot, label) \
120	LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \
121	CMP_WORD "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \
122	"beq- cr7, " __rseq_str(label) "\n\t"
123
124	#define RSEQ_ASM_OP_STORE(value, var) \
125	STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"
126
127	/* Load @var to r17 */
128	#define RSEQ_ASM_OP_R_LOAD(var) \
129	LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
130
131	/* Store r17 to @var */
132	#define RSEQ_ASM_OP_R_STORE(var) \
133	STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
134
135	/* Add @count to r17 */
136	#define RSEQ_ASM_OP_R_ADD(count) \
137	"add %%r17, %[" __rseq_str(count) "], %%r17\n\t"
138
139	/* Load (r17 + voffp) to r17 */
140	#define RSEQ_ASM_OP_R_LOADX(voffp) \
141	LOADX_WORD "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"
142
143	/* TODO: implement a faster memcpy. */
144	#define RSEQ_ASM_OP_R_MEMCPY() \
145	"cmpdi %%r19, 0\n\t" \
146	"beq 333f\n\t" \
147	"addi %%r20, %%r20, -1\n\t" \
148	"addi %%r21, %%r21, -1\n\t" \
149	"222:\n\t" \
150	"lbzu %%r18, 1(%%r20)\n\t" \
151	"stbu %%r18, 1(%%r21)\n\t" \
152	"addi %%r19, %%r19, -1\n\t" \
153	"cmpdi %%r19, 0\n\t" \
154	"bne 222b\n\t" \
155	"333:\n\t" \
156
157	#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
158	STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \
159	__rseq_str(post_commit_label) ":\n\t"
160
161	#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
162	STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
163	__rseq_str(post_commit_label) ":\n\t"
164
165	static inline __attribute__((always_inline))
166	int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
167	{
168	RSEQ_INJECT_C(9)
169
170	__asm__ __volatile__ goto (
171	RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
172	/* Start rseq by storing table entry pointer into rseq_cs. */
173	RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
174	/* cmp cpuid */
175	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
176	RSEQ_INJECT_ASM(3)
177	/* cmp @v equal to @expect */
178	RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
179	RSEQ_INJECT_ASM(4)
180	#ifdef RSEQ_COMPARE_TWICE
181	/* cmp cpuid */
182	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
183	/* cmp @v equal to @expect */
184	RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
185	#endif
186	/* final store */
187	RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
188	RSEQ_INJECT_ASM(5)
189	RSEQ_ASM_DEFINE_ABORT(4, abort)
190	: /* gcc asm goto does not allow outputs */
191	: [cpu_id] "r" (cpu),
192	[current_cpu_id] "m" (__rseq_abi.cpu_id),
193	[rseq_cs] "m" (__rseq_abi.rseq_cs),
194	[v] "m" (*v),
195	[expect] "r" (expect),
196	[newv] "r" (newv)
197	RSEQ_INJECT_INPUT
198	: "memory", "cc", "r17"
199	RSEQ_INJECT_CLOBBER
200	: abort, cmpfail
201	#ifdef RSEQ_COMPARE_TWICE
202	, error1, error2
203	#endif
204	);
205	return 0;
206	abort:
207	RSEQ_INJECT_FAILED
208	return -1;
209	cmpfail:
210	return 1;
211	#ifdef RSEQ_COMPARE_TWICE
212	error1:
213	rseq_bug("cpu_id comparison failed");
214	error2:
215	rseq_bug("expected value comparison failed");
216	#endif
217	}
218
219	static inline __attribute__((always_inline))
220	int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
221	off_t voffp, intptr_t *load, int cpu)
222	{
223	RSEQ_INJECT_C(9)
224
225	__asm__ __volatile__ goto (
226	RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
227	/* Start rseq by storing table entry pointer into rseq_cs. */
228	RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
229	/* cmp cpuid */
230	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
231	RSEQ_INJECT_ASM(3)
232	/* cmp @v not equal to @expectnot */
233	RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
234	RSEQ_INJECT_ASM(4)
235	#ifdef RSEQ_COMPARE_TWICE
236	/* cmp cpuid */
237	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
238	/* cmp @v not equal to @expectnot */
239	RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
240	#endif
241	/* load the value of @v */
242	RSEQ_ASM_OP_R_LOAD(v)
243	/* store it in @load */
244	RSEQ_ASM_OP_R_STORE(load)
245	/* dereference voffp(v) */
246	RSEQ_ASM_OP_R_LOADX(voffp)
247	/* final store the value at voffp(v) */
248	RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
249	RSEQ_INJECT_ASM(5)
250	RSEQ_ASM_DEFINE_ABORT(4, abort)
251	: /* gcc asm goto does not allow outputs */
252	: [cpu_id] "r" (cpu),
253	[current_cpu_id] "m" (__rseq_abi.cpu_id),
254	[rseq_cs] "m" (__rseq_abi.rseq_cs),
255	/* final store input */
256	[v] "m" (*v),
257	[expectnot] "r" (expectnot),
258	[voffp] "b" (voffp),
259	[load] "m" (*load)
260	RSEQ_INJECT_INPUT
261	: "memory", "cc", "r17"
262	RSEQ_INJECT_CLOBBER
263	: abort, cmpfail
264	#ifdef RSEQ_COMPARE_TWICE
265	, error1, error2
266	#endif
267	);
268	return 0;
269	abort:
270	RSEQ_INJECT_FAILED
271	return -1;
272	cmpfail:
273	return 1;
274	#ifdef RSEQ_COMPARE_TWICE
275	error1:
276	rseq_bug("cpu_id comparison failed");
277	error2:
278	rseq_bug("expected value comparison failed");
279	#endif
280	}
281
282	static inline __attribute__((always_inline))
283	int rseq_addv(intptr_t *v, intptr_t count, int cpu)
284	{
285	RSEQ_INJECT_C(9)
286
287	__asm__ __volatile__ goto (
288	RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
289	/* Start rseq by storing table entry pointer into rseq_cs. */
290	RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
291	/* cmp cpuid */
292	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
293	RSEQ_INJECT_ASM(3)
294	#ifdef RSEQ_COMPARE_TWICE
295	/* cmp cpuid */
296	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
297	#endif
298	/* load the value of @v */
299	RSEQ_ASM_OP_R_LOAD(v)
300	/* add @count to it */
301	RSEQ_ASM_OP_R_ADD(count)
302	/* final store */
303	RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
304	RSEQ_INJECT_ASM(4)
305	RSEQ_ASM_DEFINE_ABORT(4, abort)
306	: /* gcc asm goto does not allow outputs */
307	: [cpu_id] "r" (cpu),
308	[current_cpu_id] "m" (__rseq_abi.cpu_id),
309	[rseq_cs] "m" (__rseq_abi.rseq_cs),
310	/* final store input */
311	[v] "m" (*v),
312	[count] "r" (count)
313	RSEQ_INJECT_INPUT
314	: "memory", "cc", "r17"
315	RSEQ_INJECT_CLOBBER
316	: abort
317	#ifdef RSEQ_COMPARE_TWICE
318	, error1
319	#endif
320	);
321	return 0;
322	abort:
323	RSEQ_INJECT_FAILED
324	return -1;
325	#ifdef RSEQ_COMPARE_TWICE
326	error1:
327	rseq_bug("cpu_id comparison failed");
328	#endif
329	}
330
331	static inline __attribute__((always_inline))
332	int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
333	intptr_t *v2, intptr_t newv2,
334	intptr_t newv, int cpu)
335	{
336	RSEQ_INJECT_C(9)
337
338	__asm__ __volatile__ goto (
339	RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
340	/* Start rseq by storing table entry pointer into rseq_cs. */
341	RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
342	/* cmp cpuid */
343	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
344	RSEQ_INJECT_ASM(3)
345	/* cmp @v equal to @expect */
346	RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
347	RSEQ_INJECT_ASM(4)
348	#ifdef RSEQ_COMPARE_TWICE
349	/* cmp cpuid */
350	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
351	/* cmp @v equal to @expect */
352	RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
353	#endif
354	/* try store */
355	RSEQ_ASM_OP_STORE(newv2, v2)
356	RSEQ_INJECT_ASM(5)
357	/* final store */
358	RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
359	RSEQ_INJECT_ASM(6)
360	RSEQ_ASM_DEFINE_ABORT(4, abort)
361	: /* gcc asm goto does not allow outputs */
362	: [cpu_id] "r" (cpu),
363	[current_cpu_id] "m" (__rseq_abi.cpu_id),
364	[rseq_cs] "m" (__rseq_abi.rseq_cs),
365	/* try store input */
366	[v2] "m" (*v2),
367	[newv2] "r" (newv2),
368	/* final store input */
369	[v] "m" (*v),
370	[expect] "r" (expect),
371	[newv] "r" (newv)
372	RSEQ_INJECT_INPUT
373	: "memory", "cc", "r17"
374	RSEQ_INJECT_CLOBBER
375	: abort, cmpfail
376	#ifdef RSEQ_COMPARE_TWICE
377	, error1, error2
378	#endif
379	);
380	return 0;
381	abort:
382	RSEQ_INJECT_FAILED
383	return -1;
384	cmpfail:
385	return 1;
386	#ifdef RSEQ_COMPARE_TWICE
387	error1:
388	rseq_bug("cpu_id comparison failed");
389	error2:
390	rseq_bug("expected value comparison failed");
391	#endif
392	}
393
394	static inline __attribute__((always_inline))
395	int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
396	intptr_t *v2, intptr_t newv2,
397	intptr_t newv, int cpu)
398	{
399	RSEQ_INJECT_C(9)
400
401	__asm__ __volatile__ goto (
402	RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
403	/* Start rseq by storing table entry pointer into rseq_cs. */
404	RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
405	/* cmp cpuid */
406	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
407	RSEQ_INJECT_ASM(3)
408	/* cmp @v equal to @expect */
409	RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
410	RSEQ_INJECT_ASM(4)
411	#ifdef RSEQ_COMPARE_TWICE
412	/* cmp cpuid */
413	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
414	/* cmp @v equal to @expect */
415	RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
416	#endif
417	/* try store */
418	RSEQ_ASM_OP_STORE(newv2, v2)
419	RSEQ_INJECT_ASM(5)
420	/* for 'release' */
421	"lwsync\n\t"
422	/* final store */
423	RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
424	RSEQ_INJECT_ASM(6)
425	RSEQ_ASM_DEFINE_ABORT(4, abort)
426	: /* gcc asm goto does not allow outputs */
427	: [cpu_id] "r" (cpu),
428	[current_cpu_id] "m" (__rseq_abi.cpu_id),
429	[rseq_cs] "m" (__rseq_abi.rseq_cs),
430	/* try store input */
431	[v2] "m" (*v2),
432	[newv2] "r" (newv2),
433	/* final store input */
434	[v] "m" (*v),
435	[expect] "r" (expect),
436	[newv] "r" (newv)
437	RSEQ_INJECT_INPUT
438	: "memory", "cc", "r17"
439	RSEQ_INJECT_CLOBBER
440	: abort, cmpfail
441	#ifdef RSEQ_COMPARE_TWICE
442	, error1, error2
443	#endif
444	);
445	return 0;
446	abort:
447	RSEQ_INJECT_FAILED
448	return -1;
449	cmpfail:
450	return 1;
451	#ifdef RSEQ_COMPARE_TWICE
452	error1:
453	rseq_bug("cpu_id comparison failed");
454	error2:
455	rseq_bug("expected value comparison failed");
456	#endif
457	}
458
459	static inline __attribute__((always_inline))
460	int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
461	intptr_t *v2, intptr_t expect2,
462	intptr_t newv, int cpu)
463	{
464	RSEQ_INJECT_C(9)
465
466	__asm__ __volatile__ goto (
467	RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
468	/* Start rseq by storing table entry pointer into rseq_cs. */
469	RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
470	/* cmp cpuid */
471	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
472	RSEQ_INJECT_ASM(3)
473	/* cmp @v equal to @expect */
474	RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
475	RSEQ_INJECT_ASM(4)
476	/* cmp @v2 equal to @expct2 */
477	RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
478	RSEQ_INJECT_ASM(5)
479	#ifdef RSEQ_COMPARE_TWICE
480	/* cmp cpuid */
481	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
482	/* cmp @v equal to @expect */
483	RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
484	/* cmp @v2 equal to @expct2 */
485	RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
486	#endif
487	/* final store */
488	RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
489	RSEQ_INJECT_ASM(6)
490	RSEQ_ASM_DEFINE_ABORT(4, abort)
491	: /* gcc asm goto does not allow outputs */
492	: [cpu_id] "r" (cpu),
493	[current_cpu_id] "m" (__rseq_abi.cpu_id),
494	[rseq_cs] "m" (__rseq_abi.rseq_cs),
495	/* cmp2 input */
496	[v2] "m" (*v2),
497	[expect2] "r" (expect2),
498	/* final store input */
499	[v] "m" (*v),
500	[expect] "r" (expect),
501	[newv] "r" (newv)
502	RSEQ_INJECT_INPUT
503	: "memory", "cc", "r17"
504	RSEQ_INJECT_CLOBBER
505	: abort, cmpfail
506	#ifdef RSEQ_COMPARE_TWICE
507	, error1, error2, error3
508	#endif
509	);
510	return 0;
511	abort:
512	RSEQ_INJECT_FAILED
513	return -1;
514	cmpfail:
515	return 1;
516	#ifdef RSEQ_COMPARE_TWICE
517	error1:
518	rseq_bug("cpu_id comparison failed");
519	error2:
520	rseq_bug("1st expected value comparison failed");
521	error3:
522	rseq_bug("2nd expected value comparison failed");
523	#endif
524	}
525
526	static inline __attribute__((always_inline))
527	int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
528	void dst, void src, size_t len,
529	intptr_t newv, int cpu)
530	{
531	RSEQ_INJECT_C(9)
532
533	__asm__ __volatile__ goto (
534	RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
535	/* setup for mempcy */
536	"mr %%r19, %[len]\n\t"
537	"mr %%r20, %[src]\n\t"
538	"mr %%r21, %[dst]\n\t"
539	/* Start rseq by storing table entry pointer into rseq_cs. */
540	RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
541	/* cmp cpuid */
542	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
543	RSEQ_INJECT_ASM(3)
544	/* cmp @v equal to @expect */
545	RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
546	RSEQ_INJECT_ASM(4)
547	#ifdef RSEQ_COMPARE_TWICE
548	/* cmp cpuid */
549	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
550	/* cmp @v equal to @expect */
551	RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
552	#endif
553	/* try memcpy */
554	RSEQ_ASM_OP_R_MEMCPY()
555	RSEQ_INJECT_ASM(5)
556	/* final store */
557	RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
558	RSEQ_INJECT_ASM(6)
559	/* teardown */
560	RSEQ_ASM_DEFINE_ABORT(4, abort)
561	: /* gcc asm goto does not allow outputs */
562	: [cpu_id] "r" (cpu),
563	[current_cpu_id] "m" (__rseq_abi.cpu_id),
564	[rseq_cs] "m" (__rseq_abi.rseq_cs),
565	/* final store input */
566	[v] "m" (*v),
567	[expect] "r" (expect),
568	[newv] "r" (newv),
569	/* try memcpy input */
570	[dst] "r" (dst),
571	[src] "r" (src),
572	[len] "r" (len)
573	RSEQ_INJECT_INPUT
574	: "memory", "cc", "r17", "r18", "r19", "r20", "r21"
575	RSEQ_INJECT_CLOBBER
576	: abort, cmpfail
577	#ifdef RSEQ_COMPARE_TWICE
578	, error1, error2
579	#endif
580	);
581	return 0;
582	abort:
583	RSEQ_INJECT_FAILED
584	return -1;
585	cmpfail:
586	return 1;
587	#ifdef RSEQ_COMPARE_TWICE
588	error1:
589	rseq_bug("cpu_id comparison failed");
590	error2:
591	rseq_bug("expected value comparison failed");
592	#endif
593	}
594
595	static inline __attribute__((always_inline))
596	int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
597	void dst, void src, size_t len,
598	intptr_t newv, int cpu)
599	{
600	RSEQ_INJECT_C(9)
601
602	__asm__ __volatile__ goto (
603	RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
604	/* setup for mempcy */
605	"mr %%r19, %[len]\n\t"
606	"mr %%r20, %[src]\n\t"
607	"mr %%r21, %[dst]\n\t"
608	/* Start rseq by storing table entry pointer into rseq_cs. */
609	RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
610	/* cmp cpuid */
611	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
612	RSEQ_INJECT_ASM(3)
613	/* cmp @v equal to @expect */
614	RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
615	RSEQ_INJECT_ASM(4)
616	#ifdef RSEQ_COMPARE_TWICE
617	/* cmp cpuid */
618	RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
619	/* cmp @v equal to @expect */
620	RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
621	#endif
622	/* try memcpy */
623	RSEQ_ASM_OP_R_MEMCPY()
624	RSEQ_INJECT_ASM(5)
625	/* for 'release' */
626	"lwsync\n\t"
627	/* final store */
628	RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
629	RSEQ_INJECT_ASM(6)
630	/* teardown */
631	RSEQ_ASM_DEFINE_ABORT(4, abort)
632	: /* gcc asm goto does not allow outputs */
633	: [cpu_id] "r" (cpu),
634	[current_cpu_id] "m" (__rseq_abi.cpu_id),
635	[rseq_cs] "m" (__rseq_abi.rseq_cs),
636	/* final store input */
637	[v] "m" (*v),
638	[expect] "r" (expect),
639	[newv] "r" (newv),
640	/* try memcpy input */
641	[dst] "r" (dst),
642	[src] "r" (src),
643	[len] "r" (len)
644	RSEQ_INJECT_INPUT
645	: "memory", "cc", "r17", "r18", "r19", "r20", "r21"
646	RSEQ_INJECT_CLOBBER
647	: abort, cmpfail
648	#ifdef RSEQ_COMPARE_TWICE
649	, error1, error2
650	#endif
651	);
652	return 0;
653	abort:
654	RSEQ_INJECT_FAILED
655	return -1;
656	cmpfail:
657	return 1;
658	#ifdef RSEQ_COMPARE_TWICE
659	error1:
660	rseq_bug("cpu_id comparison failed");
661	error2:
662	rseq_bug("expected value comparison failed");
663	#endif
664	}
665
666	#undef STORE_WORD
667	#undef LOAD_WORD
668	#undef LOADX_WORD
669	#undef CMP_WORD
670
671	#endif /* !RSEQ_SKIP_FASTPATH */