[librseq.git] / include / rseq / rseq-mips-bits.h

/* SPDX-License-Identifier: MIT */
/* SPDX-FileCopyrightText: 2018 MIPS Tech LLC */
/* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */

/*
 * Author: Paul Burton <paul.burton@mips.com>
 */

#include "rseq-bits-template.h"

#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))

static inline __attribute__((always_inline))
int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
#endif
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
		RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		LONG_L " $4, %[v]\n\t"
		"bne $4, %[expect], %l[cmpfail]\n\t"
		RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		LONG_L " $4, %[v]\n\t"
		"bne $4, %[expect], %l[error2]\n\t"
#endif
		/* final store */
		LONG_S " %[newv], %[v]\n\t"
		"2:\n\t"
		RSEQ_INJECT_ASM(5)
		"b 5f\n\t"
		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
		"5:\n\t"
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
		  [v]			"m" (*v),
		  [expect]		"r" (expect),
		  [newv]		"r" (newv)
		  RSEQ_INJECT_INPUT
		: "$4", "memory"
		  RSEQ_INJECT_CLOBBER
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2
#endif
	);
	return 0;
abort:
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_bug("expected value comparison failed");
#endif
}

static inline __attribute__((always_inline))
int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
			       long voffp, intptr_t *load, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
#endif
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
		RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		LONG_L " $4, %[v]\n\t"
		"beq $4, %[expectnot], %l[cmpfail]\n\t"
		RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		LONG_L " $4, %[v]\n\t"
		"beq $4, %[expectnot], %l[error2]\n\t"
#endif
		LONG_S " $4, %[load]\n\t"
		LONG_ADDI " $4, %[voffp]\n\t"
		LONG_L " $4, 0($4)\n\t"
		/* final store */
		LONG_S " $4, %[v]\n\t"
		"2:\n\t"
		RSEQ_INJECT_ASM(5)
		"b 5f\n\t"
		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
		"5:\n\t"
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
		  /* final store input */
		  [v]			"m" (*v),
		  [expectnot]		"r" (expectnot),
		  [voffp]		"Ir" (voffp),
		  [load]		"m" (*load)
		  RSEQ_INJECT_INPUT
		: "$4", "memory"
		  RSEQ_INJECT_CLOBBER
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2
#endif
	);
	return 0;
abort:
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_bug("expected value comparison failed");
#endif
}

static inline __attribute__((always_inline))
int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
#endif
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
		RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
#endif
		LONG_L " $4, %[v]\n\t"
		LONG_ADDI " $4, %[count]\n\t"
		/* final store */
		LONG_S " $4, %[v]\n\t"
		"2:\n\t"
		RSEQ_INJECT_ASM(4)
		"b 5f\n\t"
		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
		"5:\n\t"
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
		  [v]			"m" (*v),
		  [count]		"Ir" (count)
		  RSEQ_INJECT_INPUT
		: "$4", "memory"
		  RSEQ_INJECT_CLOBBER
		: abort
#ifdef RSEQ_COMPARE_TWICE
		  , error1
#endif
	);
	return 0;
abort:
	RSEQ_INJECT_FAILED
	return -1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_bug("cpu_id comparison failed");
#endif
}

static inline __attribute__((always_inline))
int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
			      intptr_t *v2, intptr_t expect2,
			      intptr_t newv, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
#endif
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
		RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		LONG_L " $4, %[v]\n\t"
		"bne $4, %[expect], %l[cmpfail]\n\t"
		RSEQ_INJECT_ASM(4)
		LONG_L " $4, %[v2]\n\t"
		"bne $4, %[expect2], %l[cmpfail]\n\t"
		RSEQ_INJECT_ASM(5)
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		LONG_L " $4, %[v]\n\t"
		"bne $4, %[expect], %l[error2]\n\t"
		LONG_L " $4, %[v2]\n\t"
		"bne $4, %[expect2], %l[error3]\n\t"
#endif
		/* final store */
		LONG_S " %[newv], %[v]\n\t"
		"2:\n\t"
		RSEQ_INJECT_ASM(6)
		"b 5f\n\t"
		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
		"5:\n\t"
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
		  /* cmp2 input */
		  [v2]			"m" (*v2),
		  [expect2]		"r" (expect2),
		  /* final store input */
		  [v]			"m" (*v),
		  [expect]		"r" (expect),
		  [newv]		"r" (newv)
		  RSEQ_INJECT_INPUT
		: "$4", "memory"
		  RSEQ_INJECT_CLOBBER
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2, error3
#endif
	);
	return 0;
abort:
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_bug("1st expected value comparison failed");
error3:
	rseq_bug("2nd expected value comparison failed");
#endif
}

#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */

#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))

static inline __attribute__((always_inline))
int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
				 intptr_t *v2, intptr_t newv2,
				 intptr_t newv, int cpu)
{
	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
#endif
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
		RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		LONG_L " $4, %[v]\n\t"
		"bne $4, %[expect], %l[cmpfail]\n\t"
		RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
		LONG_L " $4, %[v]\n\t"
		"bne $4, %[expect], %l[error2]\n\t"
#endif
		/* try store */
		LONG_S " %[newv2], %[v2]\n\t"
		RSEQ_INJECT_ASM(5)
#ifdef RSEQ_TEMPLATE_MO_RELEASE
		"sync\n\t"	/* full sync provides store-release */
#endif
		/* final store */
		LONG_S " %[newv], %[v]\n\t"
		"2:\n\t"
		RSEQ_INJECT_ASM(6)
		"b 5f\n\t"
		RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
		"5:\n\t"
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
		  /* try store input */
		  [v2]			"m" (*v2),
		  [newv2]		"r" (newv2),
		  /* final store input */
		  [v]			"m" (*v),
		  [expect]		"r" (expect),
		  [newv]		"r" (newv)
		  RSEQ_INJECT_INPUT
		: "$4", "memory"
		  RSEQ_INJECT_CLOBBER
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2
#endif
	);
	return 0;
abort:
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_bug("expected value comparison failed");
#endif
}

static inline __attribute__((always_inline))
int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
				 void *dst, void *src, size_t len,
				 intptr_t newv, int cpu)
{
	uintptr_t rseq_scratch[3];

	RSEQ_INJECT_C(9)

	__asm__ __volatile__ goto (
		RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
		RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
#endif
		LONG_S " %[src], %[rseq_scratch0]\n\t"
		LONG_S "  %[dst], %[rseq_scratch1]\n\t"
		LONG_S " %[len], %[rseq_scratch2]\n\t"
		/* Start rseq by storing table entry pointer into rseq_cs. */
		RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
		RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
		RSEQ_INJECT_ASM(3)
		LONG_L " $4, %[v]\n\t"
		"bne $4, %[expect], 5f\n\t"
		RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 6f)
		LONG_L " $4, %[v]\n\t"
		"bne $4, %[expect], 7f\n\t"
#endif
		/* try memcpy */
		"beqz %[len], 333f\n\t" \
		"222:\n\t" \
		"lb   $4, 0(%[src])\n\t" \
		"sb   $4, 0(%[dst])\n\t" \
		LONG_ADDI " %[src], 1\n\t" \
		LONG_ADDI " %[dst], 1\n\t" \
		LONG_ADDI " %[len], -1\n\t" \
		"bnez %[len], 222b\n\t" \
		"333:\n\t" \
		RSEQ_INJECT_ASM(5)
#ifdef RSEQ_TEMPLATE_MO_RELEASE
		"sync\n\t"	/* full sync provides store-release */
#endif
		/* final store */
		LONG_S " %[newv], %[v]\n\t"
		"2:\n\t"
		RSEQ_INJECT_ASM(6)
		/* teardown */
		LONG_L " %[len], %[rseq_scratch2]\n\t"
		LONG_L " %[dst], %[rseq_scratch1]\n\t"
		LONG_L " %[src], %[rseq_scratch0]\n\t"
		"b 8f\n\t"
		RSEQ_ASM_DEFINE_ABORT(3, 4,
				      /* teardown */
				      LONG_L " %[len], %[rseq_scratch2]\n\t"
				      LONG_L " %[dst], %[rseq_scratch1]\n\t"
				      LONG_L " %[src], %[rseq_scratch0]\n\t",
				      abort, 1b, 2b, 4f)
		RSEQ_ASM_DEFINE_CMPFAIL(5,
					/* teardown */
					LONG_L " %[len], %[rseq_scratch2]\n\t"
					LONG_L " %[dst], %[rseq_scratch1]\n\t"
					LONG_L " %[src], %[rseq_scratch0]\n\t",
					cmpfail)
#ifdef RSEQ_COMPARE_TWICE
		RSEQ_ASM_DEFINE_CMPFAIL(6,
					/* teardown */
					LONG_L " %[len], %[rseq_scratch2]\n\t"
					LONG_L " %[dst], %[rseq_scratch1]\n\t"
					LONG_L " %[src], %[rseq_scratch0]\n\t",
					error1)
		RSEQ_ASM_DEFINE_CMPFAIL(7,
					/* teardown */
					LONG_L " %[len], %[rseq_scratch2]\n\t"
					LONG_L " %[dst], %[rseq_scratch1]\n\t"
					LONG_L " %[src], %[rseq_scratch0]\n\t",
					error2)
#endif
		"8:\n\t"
		: /* gcc asm goto does not allow outputs */
		: [cpu_id]		"r" (cpu),
		  [current_cpu_id]	"m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
		  [rseq_cs]		"m" (rseq_get_abi()->rseq_cs.arch.ptr),
		  /* final store input */
		  [v]			"m" (*v),
		  [expect]		"r" (expect),
		  [newv]		"r" (newv),
		  /* try memcpy input */
		  [dst]			"r" (dst),
		  [src]			"r" (src),
		  [len]			"r" (len),
		  [rseq_scratch0]	"m" (rseq_scratch[0]),
		  [rseq_scratch1]	"m" (rseq_scratch[1]),
		  [rseq_scratch2]	"m" (rseq_scratch[2])
		  RSEQ_INJECT_INPUT
		: "$4", "memory"
		  RSEQ_INJECT_CLOBBER
		: abort, cmpfail
#ifdef RSEQ_COMPARE_TWICE
		  , error1, error2
#endif
	);
	return 0;
abort:
	RSEQ_INJECT_FAILED
	return -1;
cmpfail:
	return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
	rseq_bug("cpu_id comparison failed");
error2:
	rseq_bug("expected value comparison failed");
#endif
}

#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
	(defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */

#include "rseq-bits-reset.h"
Commit	Line	Data
2b49bb30 MD	1	/* SPDX-License-Identifier: MIT */
	2	/* SPDX-FileCopyrightText: 2018 MIPS Tech LLC */
	3	/* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
	4
	5	/*
	6	* Author: Paul Burton <paul.burton@mips.com>
	7	*/
	8
	9	#include "rseq-bits-template.h"
	10
	11	#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
	12	(defined(RSEQ_TEMPLATE_CPU_ID) \|\| defined(RSEQ_TEMPLATE_MM_CID))
	13
	14	static inline __attribute__((always_inline))
	15	int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
	16	{
	17	RSEQ_INJECT_C(9)
	18
	19	__asm__ __volatile__ goto (
	20	RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
	21	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
	22	#ifdef RSEQ_COMPARE_TWICE
	23	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
	24	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
	25	#endif
	26	/* Start rseq by storing table entry pointer into rseq_cs. */
	27	RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
769ec9a5	28	RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
2b49bb30 MD	29	RSEQ_INJECT_ASM(3)
	30	LONG_L " $4, %[v]\n\t"
	31	"bne $4, %[expect], %l[cmpfail]\n\t"
	32	RSEQ_INJECT_ASM(4)
	33	#ifdef RSEQ_COMPARE_TWICE
769ec9a5	34	RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
2b49bb30 MD	35	LONG_L " $4, %[v]\n\t"
	36	"bne $4, %[expect], %l[error2]\n\t"
	37	#endif
	38	/* final store */
	39	LONG_S " %[newv], %[v]\n\t"
	40	"2:\n\t"
	41	RSEQ_INJECT_ASM(5)
	42	"b 5f\n\t"
	43	RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
	44	"5:\n\t"
	45	: /* gcc asm goto does not allow outputs */
	46	: [cpu_id] "r" (cpu),
	47	[current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
	48	[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
	49	[v] "m" (*v),
	50	[expect] "r" (expect),
	51	[newv] "r" (newv)
	52	RSEQ_INJECT_INPUT
	53	: "$4", "memory"
	54	RSEQ_INJECT_CLOBBER
	55	: abort, cmpfail
	56	#ifdef RSEQ_COMPARE_TWICE
	57	, error1, error2
	58	#endif
	59	);
	60	return 0;
	61	abort:
	62	RSEQ_INJECT_FAILED
	63	return -1;
	64	cmpfail:
	65	return 1;
	66	#ifdef RSEQ_COMPARE_TWICE
	67	error1:
	68	rseq_bug("cpu_id comparison failed");
	69	error2:
	70	rseq_bug("expected value comparison failed");
	71	#endif
	72	}
	73
	74	static inline __attribute__((always_inline))
	75	int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
	76	long voffp, intptr_t *load, int cpu)
	77	{
	78	RSEQ_INJECT_C(9)
	79
	80	__asm__ __volatile__ goto (
	81	RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
	82	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
	83	#ifdef RSEQ_COMPARE_TWICE
	84	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
	85	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
	86	#endif
	87	/* Start rseq by storing table entry pointer into rseq_cs. */
	88	RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
769ec9a5	89	RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
2b49bb30 MD	90	RSEQ_INJECT_ASM(3)
	91	LONG_L " $4, %[v]\n\t"
	92	"beq $4, %[expectnot], %l[cmpfail]\n\t"
	93	RSEQ_INJECT_ASM(4)
	94	#ifdef RSEQ_COMPARE_TWICE
769ec9a5	95	RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
2b49bb30 MD	96	LONG_L " $4, %[v]\n\t"
	97	"beq $4, %[expectnot], %l[error2]\n\t"
	98	#endif
	99	LONG_S " $4, %[load]\n\t"
	100	LONG_ADDI " $4, %[voffp]\n\t"
	101	LONG_L " $4, 0($4)\n\t"
	102	/* final store */
	103	LONG_S " $4, %[v]\n\t"
	104	"2:\n\t"
	105	RSEQ_INJECT_ASM(5)
	106	"b 5f\n\t"
	107	RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
	108	"5:\n\t"
	109	: /* gcc asm goto does not allow outputs */
	110	: [cpu_id] "r" (cpu),
	111	[current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
	112	[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
	113	/* final store input */
	114	[v] "m" (*v),
	115	[expectnot] "r" (expectnot),
	116	[voffp] "Ir" (voffp),
	117	[load] "m" (*load)
	118	RSEQ_INJECT_INPUT
	119	: "$4", "memory"
	120	RSEQ_INJECT_CLOBBER
	121	: abort, cmpfail
	122	#ifdef RSEQ_COMPARE_TWICE
	123	, error1, error2
	124	#endif
	125	);
	126	return 0;
	127	abort:
	128	RSEQ_INJECT_FAILED
	129	return -1;
	130	cmpfail:
	131	return 1;
	132	#ifdef RSEQ_COMPARE_TWICE
	133	error1:
	134	rseq_bug("cpu_id comparison failed");
	135	error2:
	136	rseq_bug("expected value comparison failed");
	137	#endif
	138	}
	139
	140	static inline __attribute__((always_inline))
	141	int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
	142	{
	143	RSEQ_INJECT_C(9)
	144
	145	__asm__ __volatile__ goto (
	146	RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
	147	#ifdef RSEQ_COMPARE_TWICE
	148	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
	149	#endif
	150	/* Start rseq by storing table entry pointer into rseq_cs. */
	151	RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
769ec9a5	152	RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
2b49bb30 MD	153	RSEQ_INJECT_ASM(3)
2b49bb30 MD	154	#ifdef RSEQ_COMPARE_TWICE
769ec9a5	155	RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
2b49bb30 MD	156	#endif
	157	LONG_L " $4, %[v]\n\t"
	158	LONG_ADDI " $4, %[count]\n\t"
	159	/* final store */
	160	LONG_S " $4, %[v]\n\t"
	161	"2:\n\t"
	162	RSEQ_INJECT_ASM(4)
	163	"b 5f\n\t"
	164	RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
	165	"5:\n\t"
	166	: /* gcc asm goto does not allow outputs */
	167	: [cpu_id] "r" (cpu),
	168	[current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
	169	[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
	170	[v] "m" (*v),
	171	[count] "Ir" (count)
	172	RSEQ_INJECT_INPUT
	173	: "$4", "memory"
	174	RSEQ_INJECT_CLOBBER
	175	: abort
	176	#ifdef RSEQ_COMPARE_TWICE
	177	, error1
	178	#endif
	179	);
	180	return 0;
	181	abort:
	182	RSEQ_INJECT_FAILED
	183	return -1;
	184	#ifdef RSEQ_COMPARE_TWICE
	185	error1:
	186	rseq_bug("cpu_id comparison failed");
	187	#endif
	188	}
	189
	190	static inline __attribute__((always_inline))
	191	int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
	192	intptr_t *v2, intptr_t expect2,
	193	intptr_t newv, int cpu)
	194	{
	195	RSEQ_INJECT_C(9)
	196
	197	__asm__ __volatile__ goto (
	198	RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
	199	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
	200	#ifdef RSEQ_COMPARE_TWICE
	201	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
	202	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
	203	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
	204	#endif
	205	/* Start rseq by storing table entry pointer into rseq_cs. */
	206	RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
769ec9a5	207	RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
2b49bb30 MD	208	RSEQ_INJECT_ASM(3)
	209	LONG_L " $4, %[v]\n\t"
	210	"bne $4, %[expect], %l[cmpfail]\n\t"
	211	RSEQ_INJECT_ASM(4)
	212	LONG_L " $4, %[v2]\n\t"
	213	"bne $4, %[expect2], %l[cmpfail]\n\t"
	214	RSEQ_INJECT_ASM(5)
	215	#ifdef RSEQ_COMPARE_TWICE
769ec9a5	216	RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
2b49bb30 MD	217	LONG_L " $4, %[v]\n\t"
	218	"bne $4, %[expect], %l[error2]\n\t"
	219	LONG_L " $4, %[v2]\n\t"
	220	"bne $4, %[expect2], %l[error3]\n\t"
	221	#endif
	222	/* final store */
	223	LONG_S " %[newv], %[v]\n\t"
	224	"2:\n\t"
	225	RSEQ_INJECT_ASM(6)
	226	"b 5f\n\t"
	227	RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
	228	"5:\n\t"
	229	: /* gcc asm goto does not allow outputs */
	230	: [cpu_id] "r" (cpu),
	231	[current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
	232	[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
	233	/* cmp2 input */
	234	[v2] "m" (*v2),
	235	[expect2] "r" (expect2),
	236	/* final store input */
	237	[v] "m" (*v),
	238	[expect] "r" (expect),
	239	[newv] "r" (newv)
	240	RSEQ_INJECT_INPUT
	241	: "$4", "memory"
	242	RSEQ_INJECT_CLOBBER
	243	: abort, cmpfail
	244	#ifdef RSEQ_COMPARE_TWICE
	245	, error1, error2, error3
	246	#endif
	247	);
	248	return 0;
	249	abort:
	250	RSEQ_INJECT_FAILED
	251	return -1;
	252	cmpfail:
	253	return 1;
	254	#ifdef RSEQ_COMPARE_TWICE
	255	error1:
	256	rseq_bug("cpu_id comparison failed");
	257	error2:
	258	rseq_bug("1st expected value comparison failed");
	259	error3:
	260	rseq_bug("2nd expected value comparison failed");
	261	#endif
	262	}
	263
	264	#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
	265	(defined(RSEQ_TEMPLATE_CPU_ID) \|\| defined(RSEQ_TEMPLATE_MM_CID)) */
	266
	267	#if (defined(RSEQ_TEMPLATE_MO_RELAXED) \|\| defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
	268	(defined(RSEQ_TEMPLATE_CPU_ID) \|\| defined(RSEQ_TEMPLATE_MM_CID))
	269
	270	static inline __attribute__((always_inline))
	271	int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
	272	intptr_t *v2, intptr_t newv2,
	273	intptr_t newv, int cpu)
	274	{
	275	RSEQ_INJECT_C(9)
	276
	277	__asm__ __volatile__ goto (
	278	RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
	279	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
	280	#ifdef RSEQ_COMPARE_TWICE
281	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
282	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
283	#endif
284	/* Start rseq by storing table entry pointer into rseq_cs. */
285	RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
769ec9a5	286	RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
2b49bb30 MD	287	RSEQ_INJECT_ASM(3)
	288	LONG_L " $4, %[v]\n\t"
	289	"bne $4, %[expect], %l[cmpfail]\n\t"
	290	RSEQ_INJECT_ASM(4)
	291	#ifdef RSEQ_COMPARE_TWICE
769ec9a5	292	RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
2b49bb30 MD	293	LONG_L " $4, %[v]\n\t"
	294	"bne $4, %[expect], %l[error2]\n\t"
	295	#endif
	296	/* try store */
	297	LONG_S " %[newv2], %[v2]\n\t"
	298	RSEQ_INJECT_ASM(5)
	299	#ifdef RSEQ_TEMPLATE_MO_RELEASE
	300	"sync\n\t" /* full sync provides store-release */
	301	#endif
	302	/* final store */
	303	LONG_S " %[newv], %[v]\n\t"
	304	"2:\n\t"
	305	RSEQ_INJECT_ASM(6)
	306	"b 5f\n\t"
	307	RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
	308	"5:\n\t"
	309	: /* gcc asm goto does not allow outputs */
	310	: [cpu_id] "r" (cpu),
	311	[current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
	312	[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
	313	/* try store input */
	314	[v2] "m" (*v2),
	315	[newv2] "r" (newv2),
	316	/* final store input */
	317	[v] "m" (*v),
	318	[expect] "r" (expect),
	319	[newv] "r" (newv)
	320	RSEQ_INJECT_INPUT
	321	: "$4", "memory"
	322	RSEQ_INJECT_CLOBBER
	323	: abort, cmpfail
	324	#ifdef RSEQ_COMPARE_TWICE
	325	, error1, error2
	326	#endif
	327	);
	328	return 0;
	329	abort:
	330	RSEQ_INJECT_FAILED
	331	return -1;
	332	cmpfail:
	333	return 1;
	334	#ifdef RSEQ_COMPARE_TWICE
	335	error1:
	336	rseq_bug("cpu_id comparison failed");
	337	error2:
	338	rseq_bug("expected value comparison failed");
	339	#endif
	340	}
	341
	342	static inline __attribute__((always_inline))
	343	int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
	344	void dst, void src, size_t len,
	345	intptr_t newv, int cpu)
	346	{
	347	uintptr_t rseq_scratch[3];
	348
	349	RSEQ_INJECT_C(9)
	350
	351	__asm__ __volatile__ goto (
	352	RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
	353	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
	354	#ifdef RSEQ_COMPARE_TWICE
	355	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
	356	RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
357	#endif
358	LONG_S " %[src], %[rseq_scratch0]\n\t"
359	LONG_S " %[dst], %[rseq_scratch1]\n\t"
360	LONG_S " %[len], %[rseq_scratch2]\n\t"
361	/* Start rseq by storing table entry pointer into rseq_cs. */
362	RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
769ec9a5	363	RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
2b49bb30 MD	364	RSEQ_INJECT_ASM(3)
	365	LONG_L " $4, %[v]\n\t"
	366	"bne $4, %[expect], 5f\n\t"
	367	RSEQ_INJECT_ASM(4)
	368	#ifdef RSEQ_COMPARE_TWICE
769ec9a5	369	RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 6f)
2b49bb30 MD	370	LONG_L " $4, %[v]\n\t"
	371	"bne $4, %[expect], 7f\n\t"
	372	#endif
	373	/* try memcpy */
	374	"beqz %[len], 333f\n\t" \
	375	"222:\n\t" \
	376	"lb $4, 0(%[src])\n\t" \
	377	"sb $4, 0(%[dst])\n\t" \
	378	LONG_ADDI " %[src], 1\n\t" \
	379	LONG_ADDI " %[dst], 1\n\t" \
	380	LONG_ADDI " %[len], -1\n\t" \
	381	"bnez %[len], 222b\n\t" \
	382	"333:\n\t" \
	383	RSEQ_INJECT_ASM(5)
	384	#ifdef RSEQ_TEMPLATE_MO_RELEASE
	385	"sync\n\t" /* full sync provides store-release */
	386	#endif
	387	/* final store */
	388	LONG_S " %[newv], %[v]\n\t"
	389	"2:\n\t"
	390	RSEQ_INJECT_ASM(6)
	391	/* teardown */
	392	LONG_L " %[len], %[rseq_scratch2]\n\t"
	393	LONG_L " %[dst], %[rseq_scratch1]\n\t"
	394	LONG_L " %[src], %[rseq_scratch0]\n\t"
	395	"b 8f\n\t"
	396	RSEQ_ASM_DEFINE_ABORT(3, 4,
	397	/* teardown */
	398	LONG_L " %[len], %[rseq_scratch2]\n\t"
	399	LONG_L " %[dst], %[rseq_scratch1]\n\t"
	400	LONG_L " %[src], %[rseq_scratch0]\n\t",
	401	abort, 1b, 2b, 4f)
	402	RSEQ_ASM_DEFINE_CMPFAIL(5,
	403	/* teardown */
	404	LONG_L " %[len], %[rseq_scratch2]\n\t"
	405	LONG_L " %[dst], %[rseq_scratch1]\n\t"
	406	LONG_L " %[src], %[rseq_scratch0]\n\t",
	407	cmpfail)
	408	#ifdef RSEQ_COMPARE_TWICE
	409	RSEQ_ASM_DEFINE_CMPFAIL(6,
	410	/* teardown */
	411	LONG_L " %[len], %[rseq_scratch2]\n\t"
	412	LONG_L " %[dst], %[rseq_scratch1]\n\t"
	413	LONG_L " %[src], %[rseq_scratch0]\n\t",
	414	error1)
	415	RSEQ_ASM_DEFINE_CMPFAIL(7,
	416	/* teardown */
	417	LONG_L " %[len], %[rseq_scratch2]\n\t"
	418	LONG_L " %[dst], %[rseq_scratch1]\n\t"
	419	LONG_L " %[src], %[rseq_scratch0]\n\t",
	420	error2)
	421	#endif
	422	"8:\n\t"
	423	: /* gcc asm goto does not allow outputs */
	424	: [cpu_id] "r" (cpu),
	425	[current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
	426	[rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
	427	/* final store input */
	428	[v] "m" (*v),
	429	[expect] "r" (expect),
	430	[newv] "r" (newv),
	431	/* try memcpy input */
	432	[dst] "r" (dst),
	433	[src] "r" (src),
434	[len] "r" (len),
435	[rseq_scratch0] "m" (rseq_scratch[0]),
436	[rseq_scratch1] "m" (rseq_scratch[1]),
437	[rseq_scratch2] "m" (rseq_scratch[2])
438	RSEQ_INJECT_INPUT
439	: "$4", "memory"
440	RSEQ_INJECT_CLOBBER
441	: abort, cmpfail
442	#ifdef RSEQ_COMPARE_TWICE
443	, error1, error2
444	#endif
445	);
446	return 0;
447	abort:
448	RSEQ_INJECT_FAILED
449	return -1;
450	cmpfail:
451	return 1;
452	#ifdef RSEQ_COMPARE_TWICE
453	error1:
454	rseq_bug("cpu_id comparison failed");
455	error2:
456	rseq_bug("expected value comparison failed");
457	#endif
458	}
459
460	#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) \|\| defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
461	(defined(RSEQ_TEMPLATE_CPU_ID) \|\| defined(RSEQ_TEMPLATE_MM_CID)) */
462
463	#include "rseq-bits-reset.h"