-/* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
+/* SPDX-License-Identifier: MIT */
+/* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
+/* SPDX-FileCopyrightText: 2016-2018 Boqun Feng <boqun.feng@gmail.com> */
+
/*
* rseq-ppc.h
- *
- * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
- * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>
*/
/*
#define rseq_smp_load_acquire(p) \
__extension__ ({ \
- __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
+ rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \
rseq_smp_lwsync(); \
____p1; \
})
#define rseq_smp_store_release(p, v) \
do { \
rseq_smp_lwsync(); \
- RSEQ_WRITE_ONCE(*p, v); \
+ RSEQ_WRITE_ONCE(*(p), v); \
} while (0)
#ifdef RSEQ_SKIP_FASTPATH
#define RSEQ_LOAD_INT(arg) "lwz%U[" __rseq_str(arg) "]%X[" __rseq_str(arg) "] " /* From memory ("m" constraint) */
#define RSEQ_LOADX_LONG "ldx " /* From base register ("b" constraint) */
#define RSEQ_CMP_LONG "cmpd "
+#define RSEQ_CMP_LONG_INT "cmpdi "
#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
start_ip, post_commit_offset, abort_ip) \
#define RSEQ_LOAD_INT(arg) RSEQ_LOAD_LONG(arg) /* From memory ("m" constraint) */
#define RSEQ_LOADX_LONG "lwzx " /* From base register ("b" constraint) */
#define RSEQ_CMP_LONG "cmpw "
+#define RSEQ_CMP_LONG_INT "cmpwi "
#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
start_ip, post_commit_offset, abort_ip) \
/* TODO: implement a faster memcpy. */
#define RSEQ_ASM_OP_R_MEMCPY() \
- "cmpdi %%r19, 0\n\t" \
+ RSEQ_CMP_LONG_INT "%%r19, 0\n\t" \
"beq 333f\n\t" \
"addi %%r20, %%r20, -1\n\t" \
"addi %%r21, %%r21, -1\n\t" \
"lbzu %%r18, 1(%%r20)\n\t" \
"stbu %%r18, 1(%%r21)\n\t" \
"addi %%r19, %%r19, -1\n\t" \
- "cmpdi %%r19, 0\n\t" \
+ RSEQ_CMP_LONG_INT "%%r19, 0\n\t" \
"bne 222b\n\t" \
"333:\n\t" \
: /* gcc asm goto does not allow outputs */
: [cpu_id] "r" (cpu),
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
- [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
+ [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
[v] "m" (*v),
[expect] "r" (expect),
[newv] "r" (newv)
, error1, error2
#endif
);
+ rseq_after_asm_goto();
return 0;
abort:
+ rseq_after_asm_goto();
RSEQ_INJECT_FAILED
return -1;
cmpfail:
+ rseq_after_asm_goto();
return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
+ rseq_after_asm_goto();
rseq_bug("cpu_id comparison failed");
error2:
+ rseq_after_asm_goto();
rseq_bug("expected value comparison failed");
#endif
}
static inline __attribute__((always_inline))
int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
- off_t voffp, intptr_t *load, int cpu)
+ long voffp, intptr_t *load, int cpu)
{
RSEQ_INJECT_C(9)
: /* gcc asm goto does not allow outputs */
: [cpu_id] "r" (cpu),
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
- [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
+ [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
/* final store input */
[v] "m" (*v),
[expectnot] "r" (expectnot),
, error1, error2
#endif
);
+ rseq_after_asm_goto();
return 0;
abort:
+ rseq_after_asm_goto();
RSEQ_INJECT_FAILED
return -1;
cmpfail:
+ rseq_after_asm_goto();
return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
+ rseq_after_asm_goto();
rseq_bug("cpu_id comparison failed");
error2:
+ rseq_after_asm_goto();
rseq_bug("expected value comparison failed");
#endif
}
: /* gcc asm goto does not allow outputs */
: [cpu_id] "r" (cpu),
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
- [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
+ [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
/* final store input */
[v] "m" (*v),
[count] "r" (count)
, error1
#endif
);
+ rseq_after_asm_goto();
return 0;
abort:
+ rseq_after_asm_goto();
RSEQ_INJECT_FAILED
return -1;
#ifdef RSEQ_COMPARE_TWICE
error1:
+ rseq_after_asm_goto();
rseq_bug("cpu_id comparison failed");
#endif
}
: /* gcc asm goto does not allow outputs */
: [cpu_id] "r" (cpu),
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
- [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
+ [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
/* try store input */
[v2] "m" (*v2),
[newv2] "r" (newv2),
, error1, error2
#endif
);
+ rseq_after_asm_goto();
return 0;
abort:
+ rseq_after_asm_goto();
RSEQ_INJECT_FAILED
return -1;
cmpfail:
+ rseq_after_asm_goto();
return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
+ rseq_after_asm_goto();
rseq_bug("cpu_id comparison failed");
error2:
+ rseq_after_asm_goto();
rseq_bug("expected value comparison failed");
#endif
}
: /* gcc asm goto does not allow outputs */
: [cpu_id] "r" (cpu),
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
- [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
+ [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
/* try store input */
[v2] "m" (*v2),
[newv2] "r" (newv2),
, error1, error2
#endif
);
+ rseq_after_asm_goto();
return 0;
abort:
+ rseq_after_asm_goto();
RSEQ_INJECT_FAILED
return -1;
cmpfail:
+ rseq_after_asm_goto();
return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
+ rseq_after_asm_goto();
rseq_bug("cpu_id comparison failed");
error2:
+ rseq_after_asm_goto();
rseq_bug("expected value comparison failed");
#endif
}
: /* gcc asm goto does not allow outputs */
: [cpu_id] "r" (cpu),
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
- [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
+ [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
/* cmp2 input */
[v2] "m" (*v2),
[expect2] "r" (expect2),
, error1, error2, error3
#endif
);
+ rseq_after_asm_goto();
return 0;
abort:
+ rseq_after_asm_goto();
RSEQ_INJECT_FAILED
return -1;
cmpfail:
+ rseq_after_asm_goto();
return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
+ rseq_after_asm_goto();
rseq_bug("cpu_id comparison failed");
error2:
+ rseq_after_asm_goto();
rseq_bug("1st expected value comparison failed");
error3:
+ rseq_after_asm_goto();
rseq_bug("2nd expected value comparison failed");
#endif
}
: /* gcc asm goto does not allow outputs */
: [cpu_id] "r" (cpu),
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
- [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
+ [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
/* final store input */
[v] "m" (*v),
[expect] "r" (expect),
, error1, error2
#endif
);
+ rseq_after_asm_goto();
return 0;
abort:
+ rseq_after_asm_goto();
RSEQ_INJECT_FAILED
return -1;
cmpfail:
+ rseq_after_asm_goto();
return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
+ rseq_after_asm_goto();
rseq_bug("cpu_id comparison failed");
error2:
+ rseq_after_asm_goto();
rseq_bug("expected value comparison failed");
#endif
}
: /* gcc asm goto does not allow outputs */
: [cpu_id] "r" (cpu),
[current_cpu_id] "m" (rseq_get_abi()->cpu_id),
- [rseq_cs] "m" (rseq_get_abi()->rseq_cs),
+ [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
/* final store input */
[v] "m" (*v),
[expect] "r" (expect),
, error1, error2
#endif
);
+ rseq_after_asm_goto();
return 0;
abort:
+ rseq_after_asm_goto();
RSEQ_INJECT_FAILED
return -1;
cmpfail:
+ rseq_after_asm_goto();
return 1;
#ifdef RSEQ_COMPARE_TWICE
error1:
+ rseq_after_asm_goto();
rseq_bug("cpu_id comparison failed");
error2:
+ rseq_after_asm_goto();
rseq_bug("expected value comparison failed");
#endif
}
-#undef RSEQ_STORE_LONG
-#undef RSEQ_LOAD_LONG
-#undef RSEQ_LOADX_LONG
-#undef RSEQ_CMP_LONG
-
#endif /* !RSEQ_SKIP_FASTPATH */