[librseq.git] / include / rseq / rseq-ppc-bits.h

/* SPDX-License-Identifier: MIT */
/* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
/* SPDX-FileCopyrightText: 2016-2018 Boqun Feng <boqun.feng@gmail.com> */

/*
 * rseq-ppc-bits.h
 */

#include "rseq-bits-template.h"

#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))

static inline __attribute__((always_inline))
int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
#endif
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
		/* cmp cpuid */
		RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		/* cmp @v equal to @expect */
		RSEQ_ASM_OP_CBNE(v, expect, %l[cmpfail])
		RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
		/* cmp cpuid */
		RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		/* cmp @v equal to @expect */
		RSEQ_ASM_OP_CBNE(v, expect, %l[error2])
#endif
		/* final store */
		RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
		RSEQ_INJECT_ASM(5)
		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
		  [v]			"m" (*v),
		  [expect]		"r" (expect),
		  [newv]		"r" (newv)
		  RSEQ_INJECT_INPUT
		: "memory", "cc", "r17"
		  RSEQ_INJECT_CLOBBER
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2
#endif
	);
	rseq_after_asm_goto();
	return 0;
abort:
	rseq_after_asm_goto();
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	rseq_after_asm_goto();
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_after_asm_goto();
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_after_asm_goto();
	rseq_bug("expected value comparison failed");
#endif
}

static inline __attribute__((always_inline))
int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
			       long voffp, intptr_t *load, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
#endif
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
		/* cmp cpuid */
		RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		/* cmp @v not equal to @expectnot */
		RSEQ_ASM_OP_CBEQ(v, expectnot, %l[cmpfail])
		RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
		/* cmp cpuid */
		RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		/* cmp @v not equal to @expectnot */
		RSEQ_ASM_OP_CBEQ(v, expectnot, %l[error2])
#endif
		/* load the value of @v */
		RSEQ_ASM_OP_R_LOAD(v)
		/* store it in @load */
		RSEQ_ASM_OP_R_STORE(load)
		/* dereference voffp(v) */
		RSEQ_ASM_OP_R_LOADX(voffp)
		/* final store the value at voffp(v) */
		RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
		RSEQ_INJECT_ASM(5)
		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
		  /* final store input */
		  [v]			"m" (*v),
		  [expectnot]		"r" (expectnot),
		  [voffp]		"b" (voffp),
		  [load]		"m" (*load)
		  RSEQ_INJECT_INPUT
		: "memory", "cc", "r17"
		  RSEQ_INJECT_CLOBBER
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2
#endif
	);
	rseq_after_asm_goto();
	return 0;
abort:
	rseq_after_asm_goto();
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	rseq_after_asm_goto();
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_after_asm_goto();
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_after_asm_goto();
	rseq_bug("expected value comparison failed");
#endif
}

static inline __attribute__((always_inline))
int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
#endif
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
		/* cmp cpuid */
		RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
#ifdef RSEQ_COMPARE_TWICE
		/* cmp cpuid */
		RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
#endif
		/* load the value of @v */
		RSEQ_ASM_OP_R_LOAD(v)
		/* add @count to it */
		RSEQ_ASM_OP_R_ADD(count)
		/* final store */
		RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
		RSEQ_INJECT_ASM(4)
		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
		  /* final store input */
		  [v]			"m" (*v),
		  [count]		"r" (count)
		  RSEQ_INJECT_INPUT
		: "memory", "cc", "r17"
		  RSEQ_INJECT_CLOBBER
		: abort
#ifdef RSEQ_COMPARE_TWICE
		  , error1
#endif
	);
	rseq_after_asm_goto();
	return 0;
abort:
	rseq_after_asm_goto();
	RSEQ_INJECT_FAILED
	return -1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_after_asm_goto();
	rseq_bug("cpu_id comparison failed");
#endif
}

static inline __attribute__((always_inline))
int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
			      intptr_t *v2, intptr_t expect2,
			      intptr_t newv, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
#endif
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
		/* cmp cpuid */
		RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		/* cmp @v equal to @expect */
		RSEQ_ASM_OP_CBNE(v, expect, %l[cmpfail])
		RSEQ_INJECT_ASM(4)
		/* cmp @v2 equal to @expct2 */
		RSEQ_ASM_OP_CBNE(v2, expect2, %l[cmpfail])
		RSEQ_INJECT_ASM(5)
#ifdef RSEQ_COMPARE_TWICE
		/* cmp cpuid */
		RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		/* cmp @v equal to @expect */
		RSEQ_ASM_OP_CBNE(v, expect, %l[error2])
		/* cmp @v2 equal to @expct2 */
		RSEQ_ASM_OP_CBNE(v2, expect2, %l[error3])
#endif
		/* final store */
		RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
		RSEQ_INJECT_ASM(6)
		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
		  /* cmp2 input */
		  [v2]			"m" (*v2),
		  [expect2]		"r" (expect2),
		  /* final store input */
		  [v]			"m" (*v),
		  [expect]		"r" (expect),
		  [newv]		"r" (newv)
		  RSEQ_INJECT_INPUT
		: "memory", "cc", "r17"
		  RSEQ_INJECT_CLOBBER
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2, error3
#endif
	);
	rseq_after_asm_goto();
	return 0;
abort:
	rseq_after_asm_goto();
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	rseq_after_asm_goto();
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_after_asm_goto();
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_after_asm_goto();
	rseq_bug("1st expected value comparison failed");
error3:
	rseq_after_asm_goto();
	rseq_bug("2nd expected value comparison failed");
#endif
}

#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */

#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))

static inline __attribute__((always_inline))
int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
				 intptr_t *v2, intptr_t newv2,
				 intptr_t newv, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
#endif
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
		/* cmp cpuid */
		RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		/* cmp @v equal to @expect */
		RSEQ_ASM_OP_CBNE(v, expect, %l[cmpfail])
		RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
		/* cmp cpuid */
		RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		/* cmp @v equal to @expect */
		RSEQ_ASM_OP_CBNE(v, expect, %l[error2])
#endif
		/* try store */
		RSEQ_ASM_OP_STORE(newv2, v2)
		RSEQ_INJECT_ASM(5)
#ifdef RSEQ_TEMPLATE_MO_RELEASE
		/* for 'release' */
		"lwsync\n\t"
#endif
		/* final store */
		RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
		RSEQ_INJECT_ASM(6)
		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
		  /* try store input */
		  [v2]			"m" (*v2),
		  [newv2]		"r" (newv2),
		  /* final store input */
		  [v]			"m" (*v),
		  [expect]		"r" (expect),
		  [newv]		"r" (newv)
		  RSEQ_INJECT_INPUT
		: "memory", "cc", "r17"
		  RSEQ_INJECT_CLOBBER
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2
#endif
	);
	rseq_after_asm_goto();
	return 0;
abort:
	rseq_after_asm_goto();
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	rseq_after_asm_goto();
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_after_asm_goto();
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_after_asm_goto();
	rseq_bug("expected value comparison failed");
#endif
}

static inline __attribute__((always_inline))
int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
				 void *dst, void *src, size_t len,
				 intptr_t newv, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
#endif
		/* setup for mempcy */
		"mr %%r19, %[len]\n\t"
		"mr %%r20, %[src]\n\t"
		"mr %%r21, %[dst]\n\t"
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
		/* cmp cpuid */
		RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		/* cmp @v equal to @expect */
		RSEQ_ASM_OP_CBNE(v, expect, %l[cmpfail])
		RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
		/* cmp cpuid */
		RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		/* cmp @v equal to @expect */
		RSEQ_ASM_OP_CBNE(v, expect, %l[error2])
#endif
		/* try memcpy */
		RSEQ_ASM_OP_R_MEMCPY()
		RSEQ_INJECT_ASM(5)
#ifdef RSEQ_TEMPLATE_MO_RELEASE
		/* for 'release' */
		"lwsync\n\t"
#endif
		/* final store */
		RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
		RSEQ_INJECT_ASM(6)
		/* teardown */
		RSEQ_ASM_DEFINE_ABORT(4, "", abort)
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
		  /* final store input */
		  [v]			"m" (*v),
		  [expect]		"r" (expect),
		  [newv]		"r" (newv),
		  /* try memcpy input */
		  [dst]			"r" (dst),
		  [src]			"r" (src),
		  [len]			"r" (len)
		  RSEQ_INJECT_INPUT
		: "memory", "cc", "r17", "r18", "r19", "r20", "r21"
		  RSEQ_INJECT_CLOBBER
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2
#endif
	);
	rseq_after_asm_goto();
	return 0;
abort:
	rseq_after_asm_goto();
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	rseq_after_asm_goto();
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_after_asm_goto();
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_after_asm_goto();
	rseq_bug("expected value comparison failed");
#endif
}

#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */

#include "rseq-bits-reset.h"
Commit	Line	Data
8fad586a MD	1	/* SPDX-License-Identifier: MIT */
	2	/* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
	3	/* SPDX-FileCopyrightText: 2016-2018 Boqun Feng <boqun.feng@gmail.com> */
	4
	5	/*
	6	* rseq-ppc-bits.h
	7	*/
	8
	9	#include "rseq-bits-template.h"
	10
	11	#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
	12	(defined(RSEQ_TEMPLATE_CPU_ID) \|\| defined(RSEQ_TEMPLATE_MM_CID))
	13
	14	static inline __attribute__((always_inline))
	15	int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
	16	{
	17	RSEQ_INJECT_C(9)
	18
	19	__asm__ __volatile__ goto (
	20	RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
	21	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
	22	#ifdef RSEQ_COMPARE_TWICE
	23	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
	24	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
	25	#endif
	26	/* Start rseq by storing table entry pointer into rseq_cs. */
	27	RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
	28	/* cmp cpuid */
769ec9a5	29	RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
8fad586a MD	30	RSEQ_INJECT_ASM(3)
8fad586a MD	31	/* cmp @v equal to @expect */
769ec9a5	32	RSEQ_ASM_OP_CBNE(v, expect, %l[cmpfail])
8fad586a MD	33	RSEQ_INJECT_ASM(4)
	34	#ifdef RSEQ_COMPARE_TWICE
	35	/* cmp cpuid */
769ec9a5	36	RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
8fad586a	37	/* cmp @v equal to @expect */
769ec9a5	38	RSEQ_ASM_OP_CBNE(v, expect, %l[error2])
8fad586a MD	39	#endif
	40	/* final store */
	41	RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
	42	RSEQ_INJECT_ASM(5)
ebd27573	43	RSEQ_ASM_DEFINE_ABORT(4, "", abort)
8fad586a MD	44	: /* gcc asm goto does not allow outputs */
	45	: [cpu_id] "r" (cpu),
	46	[current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
	47	[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
	48	[v] "m" (*v),
	49	[expect] "r" (expect),
	50	[newv] "r" (newv)
	51	RSEQ_INJECT_INPUT
	52	: "memory", "cc", "r17"
	53	RSEQ_INJECT_CLOBBER
	54	: abort, cmpfail
	55	#ifdef RSEQ_COMPARE_TWICE
	56	, error1, error2
	57	#endif
	58	);
	59	rseq_after_asm_goto();
	60	return 0;
	61	abort:
	62	rseq_after_asm_goto();
	63	RSEQ_INJECT_FAILED
	64	return -1;
	65	cmpfail:
	66	rseq_after_asm_goto();
	67	return 1;
	68	#ifdef RSEQ_COMPARE_TWICE
	69	error1:
	70	rseq_after_asm_goto();
	71	rseq_bug("cpu_id comparison failed");
	72	error2:
	73	rseq_after_asm_goto();
	74	rseq_bug("expected value comparison failed");
	75	#endif
	76	}
	77
	78	static inline __attribute__((always_inline))
	79	int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
	80	long voffp, intptr_t *load, int cpu)
	81	{
	82	RSEQ_INJECT_C(9)
	83
	84	__asm__ __volatile__ goto (
	85	RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
	86	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
	87	#ifdef RSEQ_COMPARE_TWICE
	88	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
	89	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
	90	#endif
	91	/* Start rseq by storing table entry pointer into rseq_cs. */
	92	RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
	93	/* cmp cpuid */
769ec9a5	94	RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
8fad586a MD	95	RSEQ_INJECT_ASM(3)
8fad586a MD	96	/* cmp @v not equal to @expectnot */
769ec9a5	97	RSEQ_ASM_OP_CBEQ(v, expectnot, %l[cmpfail])
8fad586a MD	98	RSEQ_INJECT_ASM(4)
	99	#ifdef RSEQ_COMPARE_TWICE
	100	/* cmp cpuid */
769ec9a5	101	RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
8fad586a	102	/* cmp @v not equal to @expectnot */
769ec9a5	103	RSEQ_ASM_OP_CBEQ(v, expectnot, %l[error2])
8fad586a MD	104	#endif
	105	/* load the value of @v */
	106	RSEQ_ASM_OP_R_LOAD(v)
	107	/* store it in @load */
	108	RSEQ_ASM_OP_R_STORE(load)
	109	/* dereference voffp(v) */
	110	RSEQ_ASM_OP_R_LOADX(voffp)
	111	/* final store the value at voffp(v) */
	112	RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
	113	RSEQ_INJECT_ASM(5)
ebd27573	114	RSEQ_ASM_DEFINE_ABORT(4, "", abort)
8fad586a MD	115	: /* gcc asm goto does not allow outputs */
	116	: [cpu_id] "r" (cpu),
	117	[current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
	118	[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
	119	/* final store input */
	120	[v] "m" (*v),
	121	[expectnot] "r" (expectnot),
	122	[voffp] "b" (voffp),
	123	[load] "m" (*load)
	124	RSEQ_INJECT_INPUT
	125	: "memory", "cc", "r17"
	126	RSEQ_INJECT_CLOBBER
	127	: abort, cmpfail
	128	#ifdef RSEQ_COMPARE_TWICE
	129	, error1, error2
	130	#endif
	131	);
	132	rseq_after_asm_goto();
	133	return 0;
	134	abort:
	135	rseq_after_asm_goto();
	136	RSEQ_INJECT_FAILED
	137	return -1;
	138	cmpfail:
	139	rseq_after_asm_goto();
	140	return 1;
	141	#ifdef RSEQ_COMPARE_TWICE
	142	error1:
	143	rseq_after_asm_goto();
	144	rseq_bug("cpu_id comparison failed");
	145	error2:
	146	rseq_after_asm_goto();
	147	rseq_bug("expected value comparison failed");
	148	#endif
	149	}
	150
	151	static inline __attribute__((always_inline))
	152	int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
	153	{
	154	RSEQ_INJECT_C(9)
	155
	156	__asm__ __volatile__ goto (
	157	RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
	158	#ifdef RSEQ_COMPARE_TWICE
	159	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
	160	#endif
	161	/* Start rseq by storing table entry pointer into rseq_cs. */
	162	RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
	163	/* cmp cpuid */
769ec9a5	164	RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
8fad586a MD	165	RSEQ_INJECT_ASM(3)
	166	#ifdef RSEQ_COMPARE_TWICE
	167	/* cmp cpuid */
769ec9a5	168	RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
8fad586a MD	169	#endif
	170	/* load the value of @v */
	171	RSEQ_ASM_OP_R_LOAD(v)
	172	/* add @count to it */
	173	RSEQ_ASM_OP_R_ADD(count)
	174	/* final store */
	175	RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
	176	RSEQ_INJECT_ASM(4)
ebd27573	177	RSEQ_ASM_DEFINE_ABORT(4, "", abort)
8fad586a MD	178	: /* gcc asm goto does not allow outputs */
	179	: [cpu_id] "r" (cpu),
	180	[current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
	181	[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
	182	/* final store input */
	183	[v] "m" (*v),
	184	[count] "r" (count)
	185	RSEQ_INJECT_INPUT
	186	: "memory", "cc", "r17"
	187	RSEQ_INJECT_CLOBBER
	188	: abort
	189	#ifdef RSEQ_COMPARE_TWICE
	190	, error1
	191	#endif
	192	);
	193	rseq_after_asm_goto();
	194	return 0;
	195	abort:
	196	rseq_after_asm_goto();
	197	RSEQ_INJECT_FAILED
	198	return -1;
	199	#ifdef RSEQ_COMPARE_TWICE
	200	error1:
	201	rseq_after_asm_goto();
	202	rseq_bug("cpu_id comparison failed");
	203	#endif
	204	}
	205
	206	static inline __attribute__((always_inline))
	207	int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
	208	intptr_t *v2, intptr_t expect2,
	209	intptr_t newv, int cpu)
	210	{
	211	RSEQ_INJECT_C(9)
	212
	213	__asm__ __volatile__ goto (
	214	RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
	215	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
	216	#ifdef RSEQ_COMPARE_TWICE
	217	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
	218	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
	219	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
	220	#endif
	221	/* Start rseq by storing table entry pointer into rseq_cs. */
	222	RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
	223	/* cmp cpuid */
769ec9a5	224	RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
8fad586a MD	225	RSEQ_INJECT_ASM(3)
8fad586a MD	226	/* cmp @v equal to @expect */
769ec9a5	227	RSEQ_ASM_OP_CBNE(v, expect, %l[cmpfail])
8fad586a MD	228	RSEQ_INJECT_ASM(4)
8fad586a MD	229	/* cmp @v2 equal to @expct2 */
769ec9a5	230	RSEQ_ASM_OP_CBNE(v2, expect2, %l[cmpfail])
8fad586a MD	231	RSEQ_INJECT_ASM(5)
	232	#ifdef RSEQ_COMPARE_TWICE
	233	/* cmp cpuid */
769ec9a5	234	RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
8fad586a	235	/* cmp @v equal to @expect */
769ec9a5	236	RSEQ_ASM_OP_CBNE(v, expect, %l[error2])
8fad586a	237	/* cmp @v2 equal to @expct2 */
769ec9a5	238	RSEQ_ASM_OP_CBNE(v2, expect2, %l[error3])
8fad586a MD	239	#endif
	240	/* final store */
	241	RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
	242	RSEQ_INJECT_ASM(6)
ebd27573	243	RSEQ_ASM_DEFINE_ABORT(4, "", abort)
8fad586a MD	244	: /* gcc asm goto does not allow outputs */
	245	: [cpu_id] "r" (cpu),
	246	[current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
	247	[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
	248	/* cmp2 input */
	249	[v2] "m" (*v2),
	250	[expect2] "r" (expect2),
	251	/* final store input */
	252	[v] "m" (*v),
	253	[expect] "r" (expect),
	254	[newv] "r" (newv)
	255	RSEQ_INJECT_INPUT
	256	: "memory", "cc", "r17"
	257	RSEQ_INJECT_CLOBBER
	258	: abort, cmpfail
	259	#ifdef RSEQ_COMPARE_TWICE
	260	, error1, error2, error3
	261	#endif
	262	);
	263	rseq_after_asm_goto();
	264	return 0;
	265	abort:
	266	rseq_after_asm_goto();
	267	RSEQ_INJECT_FAILED
	268	return -1;
	269	cmpfail:
	270	rseq_after_asm_goto();
	271	return 1;
	272	#ifdef RSEQ_COMPARE_TWICE
	273	error1:
	274	rseq_after_asm_goto();
	275	rseq_bug("cpu_id comparison failed");
	276	error2:
	277	rseq_after_asm_goto();
	278	rseq_bug("1st expected value comparison failed");
	279	error3:
	280	rseq_after_asm_goto();
	281	rseq_bug("2nd expected value comparison failed");
	282	#endif
	283	}
	284
	285	#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
	286	(defined(RSEQ_TEMPLATE_CPU_ID) \|\| defined(RSEQ_TEMPLATE_MM_CID)) */
	287
	288	#if (defined(RSEQ_TEMPLATE_MO_RELAXED) \|\| defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
	289	(defined(RSEQ_TEMPLATE_CPU_ID) \|\| defined(RSEQ_TEMPLATE_MM_CID))
	290
	291	static inline __attribute__((always_inline))
	292	int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
	293	intptr_t *v2, intptr_t newv2,
	294	intptr_t newv, int cpu)
	295	{
	296	RSEQ_INJECT_C(9)
	297
	298	__asm__ __volatile__ goto (
	299	RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
	300	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
	301	#ifdef RSEQ_COMPARE_TWICE
	302	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
	303	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
	304	#endif
	305	/* Start rseq by storing table entry pointer into rseq_cs. */
	306	RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
	307	/* cmp cpuid */
769ec9a5	308	RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
8fad586a MD	309	RSEQ_INJECT_ASM(3)
8fad586a MD	310	/* cmp @v equal to @expect */
769ec9a5	311	RSEQ_ASM_OP_CBNE(v, expect, %l[cmpfail])
8fad586a MD	312	RSEQ_INJECT_ASM(4)
	313	#ifdef RSEQ_COMPARE_TWICE
	314	/* cmp cpuid */
769ec9a5	315	RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
8fad586a	316	/* cmp @v equal to @expect */
769ec9a5	317	RSEQ_ASM_OP_CBNE(v, expect, %l[error2])
8fad586a MD	318	#endif
	319	/* try store */
	320	RSEQ_ASM_OP_STORE(newv2, v2)
	321	RSEQ_INJECT_ASM(5)
	322	#ifdef RSEQ_TEMPLATE_MO_RELEASE
	323	/* for 'release' */
	324	"lwsync\n\t"
	325	#endif
	326	/* final store */
	327	RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
	328	RSEQ_INJECT_ASM(6)
ebd27573	329	RSEQ_ASM_DEFINE_ABORT(4, "", abort)
8fad586a MD	330	: /* gcc asm goto does not allow outputs */
	331	: [cpu_id] "r" (cpu),
	332	[current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
	333	[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
	334	/* try store input */
	335	[v2] "m" (*v2),
	336	[newv2] "r" (newv2),
	337	/* final store input */
	338	[v] "m" (*v),
	339	[expect] "r" (expect),
	340	[newv] "r" (newv)
	341	RSEQ_INJECT_INPUT
	342	: "memory", "cc", "r17"
	343	RSEQ_INJECT_CLOBBER
	344	: abort, cmpfail
	345	#ifdef RSEQ_COMPARE_TWICE
	346	, error1, error2
	347	#endif
	348	);
	349	rseq_after_asm_goto();
	350	return 0;
	351	abort:
	352	rseq_after_asm_goto();
	353	RSEQ_INJECT_FAILED
	354	return -1;
	355	cmpfail:
	356	rseq_after_asm_goto();
	357	return 1;
	358	#ifdef RSEQ_COMPARE_TWICE
	359	error1:
	360	rseq_after_asm_goto();
	361	rseq_bug("cpu_id comparison failed");
	362	error2:
	363	rseq_after_asm_goto();
	364	rseq_bug("expected value comparison failed");
	365	#endif
	366	}
	367
	368	static inline __attribute__((always_inline))
	369	int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
	370	void dst, void src, size_t len,
	371	intptr_t newv, int cpu)
	372	{
	373	RSEQ_INJECT_C(9)
	374
	375	__asm__ __volatile__ goto (
	376	RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
	377	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
	378	#ifdef RSEQ_COMPARE_TWICE
	379	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
	380	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
	381	#endif
	382	/* setup for mempcy */
	383	"mr %%r19, %[len]\n\t"
	384	"mr %%r20, %[src]\n\t"
	385	"mr %%r21, %[dst]\n\t"
	386	/* Start rseq by storing table entry pointer into rseq_cs. */
	387	RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
	388	/* cmp cpuid */
769ec9a5	389	RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
8fad586a MD	390	RSEQ_INJECT_ASM(3)
8fad586a MD	391	/* cmp @v equal to @expect */
769ec9a5	392	RSEQ_ASM_OP_CBNE(v, expect, %l[cmpfail])
8fad586a MD	393	RSEQ_INJECT_ASM(4)
	394	#ifdef RSEQ_COMPARE_TWICE
	395	/* cmp cpuid */
769ec9a5	396	RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
8fad586a	397	/* cmp @v equal to @expect */
769ec9a5	398	RSEQ_ASM_OP_CBNE(v, expect, %l[error2])
8fad586a MD	399	#endif
	400	/* try memcpy */
	401	RSEQ_ASM_OP_R_MEMCPY()
	402	RSEQ_INJECT_ASM(5)
	403	#ifdef RSEQ_TEMPLATE_MO_RELEASE
	404	/* for 'release' */
	405	"lwsync\n\t"
	406	#endif
	407	/* final store */
	408	RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
	409	RSEQ_INJECT_ASM(6)
	410	/* teardown */
ebd27573	411	RSEQ_ASM_DEFINE_ABORT(4, "", abort)
8fad586a MD	412	: /* gcc asm goto does not allow outputs */
	413	: [cpu_id] "r" (cpu),
	414	[current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
	415	[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
	416	/* final store input */
	417	[v] "m" (*v),
	418	[expect] "r" (expect),
	419	[newv] "r" (newv),
	420	/* try memcpy input */
	421	[dst] "r" (dst),
	422	[src] "r" (src),
	423	[len] "r" (len)
	424	RSEQ_INJECT_INPUT
	425	: "memory", "cc", "r17", "r18", "r19", "r20", "r21"
	426	RSEQ_INJECT_CLOBBER
	427	: abort, cmpfail
	428	#ifdef RSEQ_COMPARE_TWICE
	429	, error1, error2
	430	#endif
	431	);
	432	rseq_after_asm_goto();
	433	return 0;
	434	abort:
	435	rseq_after_asm_goto();
	436	RSEQ_INJECT_FAILED
	437	return -1;
	438	cmpfail:
	439	rseq_after_asm_goto();
	440	return 1;
	441	#ifdef RSEQ_COMPARE_TWICE
	442	error1:
	443	rseq_after_asm_goto();
	444	rseq_bug("cpu_id comparison failed");
	445	error2:
	446	rseq_after_asm_goto();
	447	rseq_bug("expected value comparison failed");
	448	#endif
	449	}
	450
	451	#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) \|\| defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
	452	(defined(RSEQ_TEMPLATE_CPU_ID) \|\| defined(RSEQ_TEMPLATE_MM_CID)) */
	453
	454	#include "rseq-bits-reset.h"