[librseq.git] / include / rseq / rseq-ppc.h

/* SPDX-License-Identifier: MIT */
/* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
/* SPDX-FileCopyrightText: 2016-2018 Boqun Feng <boqun.feng@gmail.com> */

/*
 * rseq-ppc.h
 */

/*
 * RSEQ_SIG is used with the following trap instruction:
 *
 * powerpc-be:    0f e5 00 0b           twui   r5,11
 * powerpc64-le:  0b 00 e5 0f           twui   r5,11
 * powerpc64-be:  0f e5 00 0b           twui   r5,11
 */

#define RSEQ_SIG	0x0fe5000b

#define rseq_smp_mb()		__asm__ __volatile__ ("sync"	::: "memory", "cc")
#define rseq_smp_lwsync()	__asm__ __volatile__ ("lwsync"	::: "memory", "cc")
#define rseq_smp_rmb()		rseq_smp_lwsync()
#define rseq_smp_wmb()		rseq_smp_lwsync()

#define rseq_smp_load_acquire(p)					\
__extension__ ({							\
	rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p));	\
	rseq_smp_lwsync();						\
	____p1;								\
})

#define rseq_smp_acquire__after_ctrl_dep()	rseq_smp_lwsync()

#define rseq_smp_store_release(p, v)					\
do {									\
	rseq_smp_lwsync();						\
	RSEQ_WRITE_ONCE(*(p), v);					\
} while (0)

/*
 * The __rseq_cs_ptr_array and __rseq_cs sections can be used by debuggers to
 * better handle single-stepping through the restartable critical sections.
 */

#ifdef __PPC64__

#define RSEQ_ASM_STORE_LONG(arg)	"std%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] "	/* To memory ("m" constraint) */
#define RSEQ_ASM_STORE_INT(arg)		"stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] "	/* To memory ("m" constraint) */
#define RSEQ_ASM_LOAD_LONG(arg)		"ld%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] "	/* From memory ("m" constraint) */
#define RSEQ_ASM_LOAD_INT(arg)		"lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] "	/* From memory ("m" constraint) */
#define RSEQ_ASM_LOADX_LONG		"ldx "							/* From base register ("b" constraint) */
#define RSEQ_ASM_CMP_LONG		"cmpd "							/* Register-to-register comparison */
#define RSEQ_ASM_CMP_LONG_INT		"cmpdi "						/* Register-to-immediate comparison */

#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,				\
			start_ip, post_commit_offset, abort_ip)			\
		".pushsection __rseq_cs, \"aw\"\n\t"				\
		".balign 32\n\t"						\
		__rseq_str(label) ":\n\t"					\
		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t"	\
		".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
		".popsection\n\t"						\
		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"			\
		".quad " __rseq_str(label) "b\n\t"				\
		".popsection\n\t"

#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)			\
		RSEQ_INJECT_ASM(1)						\
		"lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t"		\
		"ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t"	\
		"rldicr %%r17, %%r17, 32, 31\n\t"				\
		"oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t"	\
		"ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t"		\
		"std %%r17, %[" __rseq_str(rseq_cs) "]\n\t"			\
		__rseq_str(label) ":\n\t"

/*
 * Exit points of a rseq critical section consist of all instructions outside
 * of the critical section where a critical section can either branch to or
 * reach through the normal course of its execution. The abort IP and the
 * post-commit IP are already part of the __rseq_cs section and should not be
 * explicitly defined as additional exit points. Knowing all exit points is
 * useful to assist debuggers stepping over the critical section.
 */
#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)			\
		".pushsection __rseq_exit_point_array, \"aw\"\n\t"	\
		".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
		".popsection\n\t"

#else /* #ifdef __PPC64__ */

#define RSEQ_ASM_STORE_LONG(arg)	"stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] "	/* To memory ("m" constraint) */
#define RSEQ_ASM_STORE_INT(arg)		RSEQ_ASM_STORE_LONG(arg)				/* To memory ("m" constraint) */
#define RSEQ_ASM_LOAD_LONG(arg)		"lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] "	/* From memory ("m" constraint) */
#define RSEQ_ASM_LOAD_INT(arg)		RSEQ_ASM_LOAD_LONG(arg)					/* From memory ("m" constraint) */
#define RSEQ_ASM_LOADX_LONG		"lwzx "							/* From base register ("b" constraint) */
#define RSEQ_ASM_CMP_LONG		"cmpw "							/* Register-to-register comparison */
#define RSEQ_ASM_CMP_LONG_INT		"cmpwi "						/* Register-to-immediate comparison */

#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,				\
			start_ip, post_commit_offset, abort_ip)			\
		".pushsection __rseq_cs, \"aw\"\n\t"				\
		".balign 32\n\t"						\
		__rseq_str(label) ":\n\t"					\
		".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t"	\
		/* 32-bit only supported on BE */				\
		".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
		".popsection\n\t"					\
		".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"		\
		".long 0x0, " __rseq_str(label) "b\n\t"			\
		".popsection\n\t"

/*
 * Exit points of a rseq critical section consist of all instructions outside
 * of the critical section where a critical section can either branch to or
 * reach through the normal course of its execution. The abort IP and the
 * post-commit IP are already part of the __rseq_cs section and should not be
 * explicitly defined as additional exit points. Knowing all exit points is
 * useful to assist debuggers stepping over the critical section.
 */
#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)				\
		".pushsection __rseq_exit_point_array, \"aw\"\n\t"		\
		/* 32-bit only supported on BE */				\
		".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t"	\
		".popsection\n\t"

#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)			\
		RSEQ_INJECT_ASM(1)						\
		"lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t"			\
		"addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t"		\
		RSEQ_ASM_STORE_INT(rseq_cs) "%%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
		__rseq_str(label) ":\n\t"

#endif /* #ifdef __PPC64__ */

#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip)	\
		__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,		\
					(post_commit_ip - start_ip), abort_ip)

#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label)			\
		RSEQ_INJECT_ASM(2)						\
		RSEQ_ASM_LOAD_INT(current_cpu_id) "%%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \
		"cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t"		\
		"bne- cr7, " __rseq_str(label) "\n\t"

#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)			\
		".pushsection __rseq_failure, \"ax\"\n\t"			\
		".long " __rseq_str(RSEQ_SIG) "\n\t"				\
		__rseq_str(label) ":\n\t"					\
		teardown							\
		"b %l[" __rseq_str(abort_label) "]\n\t"				\
		".popsection\n\t"

/*
 * RSEQ_ASM_OPs: asm operations for rseq
 * 	RSEQ_ASM_OP_R_*: has hard-code registers in it
 * 	RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7)
 */
#define RSEQ_ASM_OP_CBNE(var, expect, label)					\
		RSEQ_ASM_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"	\
		RSEQ_ASM_CMP_LONG "cr7, %%r17, %[" __rseq_str(expect) "]\n\t"	\
		"bne- cr7, " __rseq_str(label) "\n\t"

#define RSEQ_ASM_OP_CBEQ(var, expectnot, label)					\
		RSEQ_ASM_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"	\
		RSEQ_ASM_CMP_LONG "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \
		"beq- cr7, " __rseq_str(label) "\n\t"

#define RSEQ_ASM_OP_STORE(value, var)						\
		RSEQ_ASM_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"

/* Load @var to r17 */
#define RSEQ_ASM_OP_R_LOAD(var)							\
		RSEQ_ASM_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"

/* Store r17 to @var */
#define RSEQ_ASM_OP_R_STORE(var)						\
		RSEQ_ASM_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"

/* Add @count to r17 */
#define RSEQ_ASM_OP_R_ADD(count)						\
		"add %%r17, %[" __rseq_str(count) "], %%r17\n\t"

/* Load (r17 + voffp) to r17 */
#define RSEQ_ASM_OP_R_LOADX(voffp)						\
		RSEQ_ASM_LOADX_LONG "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"

/* TODO: implement a faster memcpy. */
#define RSEQ_ASM_OP_R_BYTEWISE_MEMCPY() \
		RSEQ_ASM_CMP_LONG_INT "%%r19, 0\n\t" \
		"beq 333f\n\t" \
		"addi %%r20, %%r20, -1\n\t" \
		"addi %%r21, %%r21, -1\n\t" \
		"222:\n\t" \
		"lbzu %%r18, 1(%%r20)\n\t" \
		"stbu %%r18, 1(%%r21)\n\t" \
		"addi %%r19, %%r19, -1\n\t" \
		RSEQ_ASM_CMP_LONG_INT "%%r19, 0\n\t" \
		"bne 222b\n\t" \
		"333:\n\t" \

#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label)			\
		RSEQ_ASM_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"	\
		__rseq_str(post_commit_label) ":\n\t"

#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label)			\
		RSEQ_ASM_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
		__rseq_str(post_commit_label) ":\n\t"

/* Per-cpu-id indexing. */

#define RSEQ_TEMPLATE_INDEX_CPU_ID
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq-ppc-bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED

#define RSEQ_TEMPLATE_MO_RELEASE
#include "rseq-ppc-bits.h"
#undef RSEQ_TEMPLATE_MO_RELEASE
#undef RSEQ_TEMPLATE_INDEX_CPU_ID

/* Per-mm-cid indexing. */

#define RSEQ_TEMPLATE_INDEX_MM_CID
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq-ppc-bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED

#define RSEQ_TEMPLATE_MO_RELEASE
#include "rseq-ppc-bits.h"
#undef RSEQ_TEMPLATE_MO_RELEASE
#undef RSEQ_TEMPLATE_INDEX_MM_CID

/* APIs which are not indexed. */

#define RSEQ_TEMPLATE_INDEX_NONE
#define RSEQ_TEMPLATE_MO_RELAXED
#include "rseq-ppc-bits.h"
#undef RSEQ_TEMPLATE_MO_RELAXED
#undef RSEQ_TEMPLATE_INDEX_NONE
Commit	Line	Data
90702366	1	/* SPDX-License-Identifier: MIT */
f2d7b530 MJ	2	/* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
	3	/* SPDX-FileCopyrightText: 2016-2018 Boqun Feng <boqun.feng@gmail.com> */
	4
784b0012 MD	5	/*
784b0012 MD	6	* rseq-ppc.h
784b0012 MD	7	*/
784b0012 MD	8
0158e666 MD	9	/*
	10	* RSEQ_SIG is used with the following trap instruction:
	11	*
	12	* powerpc-be: 0f e5 00 0b twui r5,11
	13	* powerpc64-le: 0b 00 e5 0f twui r5,11
	14	* powerpc64-be: 0f e5 00 0b twui r5,11
	15	*/
	16
	17	#define RSEQ_SIG 0x0fe5000b
784b0012 MD	18
	19	#define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory", "cc")
	20	#define rseq_smp_lwsync() __asm__ __volatile__ ("lwsync" ::: "memory", "cc")
	21	#define rseq_smp_rmb() rseq_smp_lwsync()
	22	#define rseq_smp_wmb() rseq_smp_lwsync()
	23
	24	#define rseq_smp_load_acquire(p) \
	25	__extension__ ({ \
3664a718	26	rseq_unqual_scalar_typeof((p)) ____p1 = RSEQ_READ_ONCE((p)); \
784b0012 MD	27	rseq_smp_lwsync(); \
	28	____p1; \
	29	})
	30
	31	#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_lwsync()
	32
	33	#define rseq_smp_store_release(p, v) \
	34	do { \
	35	rseq_smp_lwsync(); \
826417f6	36	RSEQ_WRITE_ONCE(*(p), v); \
784b0012 MD	37	} while (0)
784b0012 MD	38
784b0012	39	/*
dd01d0fb MD	40	* The __rseq_cs_ptr_array and __rseq_cs sections can be used by debuggers to
dd01d0fb MD	41	* better handle single-stepping through the restartable critical sections.
784b0012 MD	42	*/
	43
	44	#ifdef __PPC64__
	45
d1d63a2d MD	46	#define RSEQ_ASM_STORE_LONG(arg) "std%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */
	47	#define RSEQ_ASM_STORE_INT(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */
	48	#define RSEQ_ASM_LOAD_LONG(arg) "ld%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */
	49	#define RSEQ_ASM_LOAD_INT(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */
	50	#define RSEQ_ASM_LOADX_LONG "ldx " /* From base register ("b" constraint) */
	51	#define RSEQ_ASM_CMP_LONG "cmpd " /* Register-to-register comparison */
	52	#define RSEQ_ASM_CMP_LONG_INT "cmpdi " /* Register-to-immediate comparison */
784b0012 MD	53
	54	#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
	55	start_ip, post_commit_offset, abort_ip) \
dd01d0fb	56	".pushsection __rseq_cs, \"aw\"\n\t" \
784b0012 MD	57	".balign 32\n\t" \
	58	__rseq_str(label) ":\n\t" \
	59	".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
	60	".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
dd01d0fb MD	61	".popsection\n\t" \
	62	".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
	63	".quad " __rseq_str(label) "b\n\t" \
784b0012 MD	64	".popsection\n\t"
	65
	66	#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
	67	RSEQ_INJECT_ASM(1) \
	68	"lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t" \
	69	"ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t" \
	70	"rldicr %%r17, %%r17, 32, 31\n\t" \
	71	"oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t" \
	72	"ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
	73	"std %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
	74	__rseq_str(label) ":\n\t"
	75
90d9876e MD	76	/*
90d9876e MD	77	* Exit points of a rseq critical section consist of all instructions outside
fd622cad MD	78	* of the critical section where a critical section can either branch to or
	79	* reach through the normal course of its execution. The abort IP and the
	80	* post-commit IP are already part of the __rseq_cs section and should not be
	81	* explicitly defined as additional exit points. Knowing all exit points is
	82	* useful to assist debuggers stepping over the critical section.
90d9876e MD	83	*/
	84	#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
	85	".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
	86	".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
	87	".popsection\n\t"
	88
784b0012 MD	89	#else /* #ifdef __PPC64__ */
784b0012 MD	90
d1d63a2d MD	91	#define RSEQ_ASM_STORE_LONG(arg) "stw%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* To memory ("m" constraint) */
	92	#define RSEQ_ASM_STORE_INT(arg) RSEQ_ASM_STORE_LONG(arg) /* To memory ("m" constraint) */
	93	#define RSEQ_ASM_LOAD_LONG(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */
	94	#define RSEQ_ASM_LOAD_INT(arg) RSEQ_ASM_LOAD_LONG(arg) /* From memory ("m" constraint) */
	95	#define RSEQ_ASM_LOADX_LONG "lwzx " /* From base register ("b" constraint) */
	96	#define RSEQ_ASM_CMP_LONG "cmpw " /* Register-to-register comparison */
	97	#define RSEQ_ASM_CMP_LONG_INT "cmpwi " /* Register-to-immediate comparison */
784b0012 MD	98
	99	#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
	100	start_ip, post_commit_offset, abort_ip) \
dd01d0fb	101	".pushsection __rseq_cs, \"aw\"\n\t" \
784b0012 MD	102	".balign 32\n\t" \
	103	__rseq_str(label) ":\n\t" \
	104	".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
	105	/* 32-bit only supported on BE */ \
	106	".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
dd01d0fb MD	107	".popsection\n\t" \
	108	".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
	109	".long 0x0, " __rseq_str(label) "b\n\t" \
784b0012 MD	110	".popsection\n\t"
784b0012 MD	111
90d9876e MD	112	/*
90d9876e MD	113	* Exit points of a rseq critical section consist of all instructions outside
fd622cad MD	114	* of the critical section where a critical section can either branch to or
	115	* reach through the normal course of its execution. The abort IP and the
	116	* post-commit IP are already part of the __rseq_cs section and should not be
	117	* explicitly defined as additional exit points. Knowing all exit points is
	118	* useful to assist debuggers stepping over the critical section.
90d9876e MD	119	*/
	120	#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
	121	".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
	122	/* 32-bit only supported on BE */ \
	123	".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
	124	".popsection\n\t"
	125
784b0012 MD	126	#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
	127	RSEQ_INJECT_ASM(1) \
	128	"lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t" \
	129	"addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
d1d63a2d	130	RSEQ_ASM_STORE_INT(rseq_cs) "%%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
784b0012 MD	131	__rseq_str(label) ":\n\t"
	132
	133	#endif /* #ifdef __PPC64__ */
	134
	135	#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
	136	__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
	137	(post_commit_ip - start_ip), abort_ip)
	138
769ec9a5	139	#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \
784b0012	140	RSEQ_INJECT_ASM(2) \
d1d63a2d	141	RSEQ_ASM_LOAD_INT(current_cpu_id) "%%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \
784b0012 MD	142	"cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t" \
	143	"bne- cr7, " __rseq_str(label) "\n\t"
	144
ebd27573	145	#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
784b0012 MD	146	".pushsection __rseq_failure, \"ax\"\n\t" \
	147	".long " __rseq_str(RSEQ_SIG) "\n\t" \
	148	__rseq_str(label) ":\n\t" \
ebd27573	149	teardown \
784b0012 MD	150	"b %l[" __rseq_str(abort_label) "]\n\t" \
	151	".popsection\n\t"
	152
	153	/*
	154	* RSEQ_ASM_OPs: asm operations for rseq
	155	* RSEQ_ASM_OP_R_*: has hard-code registers in it
	156	* RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7)
	157	*/
769ec9a5	158	#define RSEQ_ASM_OP_CBNE(var, expect, label) \
d1d63a2d MD	159	RSEQ_ASM_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \
d1d63a2d MD	160	RSEQ_ASM_CMP_LONG "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \
784b0012 MD	161	"bne- cr7, " __rseq_str(label) "\n\t"
784b0012 MD	162
d1d63a2d MD	163	#define RSEQ_ASM_OP_CBEQ(var, expectnot, label) \
	164	RSEQ_ASM_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \
	165	RSEQ_ASM_CMP_LONG "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \
784b0012 MD	166	"beq- cr7, " __rseq_str(label) "\n\t"
	167
	168	#define RSEQ_ASM_OP_STORE(value, var) \
d1d63a2d	169	RSEQ_ASM_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"
784b0012 MD	170
	171	/* Load @var to r17 */
	172	#define RSEQ_ASM_OP_R_LOAD(var) \
d1d63a2d	173	RSEQ_ASM_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"
784b0012 MD	174
	175	/* Store r17 to @var */
	176	#define RSEQ_ASM_OP_R_STORE(var) \
d1d63a2d	177	RSEQ_ASM_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t"
784b0012 MD	178
	179	/* Add @count to r17 */
	180	#define RSEQ_ASM_OP_R_ADD(count) \
	181	"add %%r17, %[" __rseq_str(count) "], %%r17\n\t"
	182
	183	/* Load (r17 + voffp) to r17 */
	184	#define RSEQ_ASM_OP_R_LOADX(voffp) \
d1d63a2d	185	RSEQ_ASM_LOADX_LONG "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"
784b0012 MD	186
784b0012 MD	187	/* TODO: implement a faster memcpy. */
08130179	188	#define RSEQ_ASM_OP_R_BYTEWISE_MEMCPY() \
d1d63a2d	189	RSEQ_ASM_CMP_LONG_INT "%%r19, 0\n\t" \
784b0012 MD	190	"beq 333f\n\t" \
	191	"addi %%r20, %%r20, -1\n\t" \
	192	"addi %%r21, %%r21, -1\n\t" \
	193	"222:\n\t" \
	194	"lbzu %%r18, 1(%%r20)\n\t" \
	195	"stbu %%r18, 1(%%r21)\n\t" \
	196	"addi %%r19, %%r19, -1\n\t" \
d1d63a2d	197	RSEQ_ASM_CMP_LONG_INT "%%r19, 0\n\t" \
784b0012 MD	198	"bne 222b\n\t" \
	199	"333:\n\t" \
	200
	201	#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
d1d63a2d	202	RSEQ_ASM_STORE_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \
784b0012 MD	203	__rseq_str(post_commit_label) ":\n\t"
	204
	205	#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
d1d63a2d	206	RSEQ_ASM_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
784b0012 MD	207	__rseq_str(post_commit_label) ":\n\t"
784b0012 MD	208
8fad586a MD	209	/* Per-cpu-id indexing. */
8fad586a MD	210
abf9e855	211	#define RSEQ_TEMPLATE_INDEX_CPU_ID
8fad586a MD	212	#define RSEQ_TEMPLATE_MO_RELAXED
	213	#include "rseq-ppc-bits.h"
	214	#undef RSEQ_TEMPLATE_MO_RELAXED
	215
	216	#define RSEQ_TEMPLATE_MO_RELEASE
	217	#include "rseq-ppc-bits.h"
	218	#undef RSEQ_TEMPLATE_MO_RELEASE
abf9e855	219	#undef RSEQ_TEMPLATE_INDEX_CPU_ID
8fad586a MD	220
	221	/* Per-mm-cid indexing. */
	222
abf9e855	223	#define RSEQ_TEMPLATE_INDEX_MM_CID
8fad586a MD	224	#define RSEQ_TEMPLATE_MO_RELAXED
	225	#include "rseq-ppc-bits.h"
	226	#undef RSEQ_TEMPLATE_MO_RELAXED
	227
	228	#define RSEQ_TEMPLATE_MO_RELEASE
	229	#include "rseq-ppc-bits.h"
	230	#undef RSEQ_TEMPLATE_MO_RELEASE
abf9e855	231	#undef RSEQ_TEMPLATE_INDEX_MM_CID
8fad586a	232
abf9e855	233	/* APIs which are not indexed. */
8fad586a	234
abf9e855	235	#define RSEQ_TEMPLATE_INDEX_NONE
8fad586a MD	236	#define RSEQ_TEMPLATE_MO_RELAXED
	237	#include "rseq-ppc-bits.h"
	238	#undef RSEQ_TEMPLATE_MO_RELAXED
abf9e855	239	#undef RSEQ_TEMPLATE_INDEX_NONE