From: Mathieu Desnoyers Date: Thu, 22 Feb 2024 15:42:08 +0000 (-0500) Subject: Clarify logic of RSEQ_ASM*_CMP* macros X-Git-Url: http://drtracing.org/?a=commitdiff_plain;h=769ec9a57004ba64365e001d73ad015d1636b2b2;p=librseq.git Clarify logic of RSEQ_ASM*_CMP* macros The RSEQ_ASM*_CMP* macros come from the x86 implementation which has "RSEQ_ASM_CMP_CPU_ID()" and static inline functions such as "rseq_cmpeqv_storev()". The meaning of the "cmp" here is that the critical section does _not_ abort (does not branch) if the comparison matches. But I understand how the ASM helpers that were contributed for other architectures such as "RSEQ_ASM_OP_CMPEQ()", with the same semantic of "do not branch to abort if the comparison matches" can be misleading for someone used to reading assembler on pretty much any architecture, where the conditional branch is expected to be taken if the condition matches. So what I have here in librseq is backwards. Fortunately, librseq is still just a master branch (no releases yet), and the copy in the Linux kernel selftests is internal to that selftest, so there are no stable API expectations at this stage. So I don't think the semantic of e.g. "rseq_cmpeqv_storev()" is misleading: it proceeds to do the store if the comparison matches. However, the ASM macros would benefit from a logic flip. Introduce this in a way that will allow users of the API to catch the change at compile-time. This commit applies the following remapping of the macros for added clarity: RSEQ_ASM_OP_CMPNE becomes RSEQ_ASM_OP_CBEQ (compare and branch if equal) RSEQ_ASM_OP_CMPEQ becomes RSEQ_ASM_OP_CBNE (compare and branch if not equal) RSEQ_ASM_CMP_CPU_ID becomes RSEQ_ASM_CBNE_CPU_ID (compare and branch if cpu id is not equal) Signed-off-by: Mathieu Desnoyers Change-Id: I13bff7e22569b11565a23aca87cf571431d57106 --- diff --git a/include/rseq/rseq-arm-bits.h b/include/rseq/rseq-arm-bits.h index 1adb110..2cff022 100644 --- a/include/rseq/rseq-arm-bits.h +++ b/include/rseq/rseq-arm-bits.h @@ -24,14 +24,14 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, i #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) "ldr r0, %[v]\n\t" "cmp %[expect], r0\n\t" "bne %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) "ldr r0, %[v]\n\t" "cmp %[expect], r0\n\t" "bne %l[error2]\n\t" @@ -92,14 +92,14 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t e #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) "ldr r0, %[v]\n\t" "cmp %[expectnot], r0\n\t" "beq %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) "ldr r0, %[v]\n\t" "cmp %[expectnot], r0\n\t" "beq %l[error2]\n\t" @@ -162,10 +162,10 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) #endif "ldr r0, %[v]\n\t" "add r0, %[count]\n\t" @@ -220,7 +220,7 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t ex #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) "ldr r0, %[v]\n\t" "cmp %[expect], r0\n\t" @@ -231,7 +231,7 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t ex "bne %l[cmpfail]\n\t" RSEQ_INJECT_ASM(5) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) "ldr r0, %[v]\n\t" "cmp %[expect], r0\n\t" "bne %l[error2]\n\t" @@ -309,14 +309,14 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) "ldr r0, %[v]\n\t" "cmp %[expect], r0\n\t" "bne %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) "ldr r0, %[v]\n\t" "cmp %[expect], r0\n\t" "bne %l[error2]\n\t" @@ -394,14 +394,14 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t "str %[len], %[rseq_scratch2]\n\t" /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) "ldr r0, %[v]\n\t" "cmp %[expect], r0\n\t" "bne 5f\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 6f) "ldr r0, %[v]\n\t" "cmp %[expect], r0\n\t" "bne 7f\n\t" diff --git a/include/rseq/rseq-arm.h b/include/rseq/rseq-arm.h index c4e3f55..9f4f297 100644 --- a/include/rseq/rseq-arm.h +++ b/include/rseq/rseq-arm.h @@ -114,7 +114,7 @@ do { \ "str r0, %[" __rseq_str(rseq_cs) "]\n\t" \ __rseq_str(label) ":\n\t" -#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ +#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \ RSEQ_INJECT_ASM(2) \ "ldr r0, %[" __rseq_str(current_cpu_id) "]\n\t" \ "cmp %[" __rseq_str(cpu_id) "], r0\n\t" \ diff --git a/include/rseq/rseq-arm64-bits.h b/include/rseq/rseq-arm64-bits.h index 7715251..279b016 100644 --- a/include/rseq/rseq-arm64-bits.h +++ b/include/rseq/rseq-arm64-bits.h @@ -24,13 +24,13 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, i RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) #endif RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) + RSEQ_ASM_OP_CBNE(v, expect, %l[cmpfail]) RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_OP_CBNE(v, expect, %l[error2]) #endif RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) RSEQ_INJECT_ASM(5) @@ -82,13 +82,13 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t e RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) #endif RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) + RSEQ_ASM_OP_CBEQ(v, expectnot, %l[cmpfail]) RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_OP_CBEQ(v, expectnot, %l[error2]) #endif RSEQ_ASM_OP_R_LOAD(v) RSEQ_ASM_OP_R_STORE(load) @@ -141,10 +141,10 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) #endif RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) #endif RSEQ_ASM_OP_R_LOAD(v) RSEQ_ASM_OP_R_ADD(count) @@ -193,16 +193,16 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t ex RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3]) #endif RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) + RSEQ_ASM_OP_CBNE(v, expect, %l[cmpfail]) RSEQ_INJECT_ASM(4) - RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) + RSEQ_ASM_OP_CBNE(v2, expect2, %l[cmpfail]) RSEQ_INJECT_ASM(5) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) - RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_OP_CBNE(v, expect, %l[error2]) + RSEQ_ASM_OP_CBNE(v2, expect2, %l[error3]) #endif RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) RSEQ_INJECT_ASM(6) @@ -266,13 +266,13 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) #endif RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) + RSEQ_ASM_OP_CBNE(v, expect, %l[cmpfail]) RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_OP_CBNE(v, expect, %l[error2]) #endif RSEQ_ASM_OP_STORE(newv2, v2) RSEQ_INJECT_ASM(5) @@ -333,13 +333,13 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) #endif RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) + RSEQ_ASM_OP_CBNE(v, expect, %l[cmpfail]) RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_OP_CBNE(v, expect, %l[error2]) #endif RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) RSEQ_INJECT_ASM(5) diff --git a/include/rseq/rseq-arm64.h b/include/rseq/rseq-arm64.h index c9d32fa..dc3b23b 100644 --- a/include/rseq/rseq-arm64.h +++ b/include/rseq/rseq-arm64.h @@ -151,27 +151,27 @@ do { \ RSEQ_ASM_OP_STORE_RELEASE(value, var) \ __rseq_str(post_commit_label) ":\n" -#define RSEQ_ASM_OP_CMPEQ(var, expect, label) \ +#define RSEQ_ASM_OP_CBNE(var, expect, label) \ " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \ " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \ ", %[" __rseq_str(expect) "]\n" \ " cbnz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n" -#define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \ +#define RSEQ_ASM_OP_CBNE32(var, expect, label) \ " ldr " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n" \ " sub " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32 \ ", %w[" __rseq_str(expect) "]\n" \ " cbnz " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n" -#define RSEQ_ASM_OP_CMPNE(var, expect, label) \ +#define RSEQ_ASM_OP_CBEQ(var, expect, label) \ " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \ " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \ ", %[" __rseq_str(expect) "]\n" \ " cbz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n" -#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ +#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \ RSEQ_INJECT_ASM(2) \ - RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label) + RSEQ_ASM_OP_CBNE32(current_cpu_id, cpu_id, label) #define RSEQ_ASM_OP_R_LOAD(var) \ " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" diff --git a/include/rseq/rseq-mips-bits.h b/include/rseq/rseq-mips-bits.h index c2f7a68..a025bde 100644 --- a/include/rseq/rseq-mips-bits.h +++ b/include/rseq/rseq-mips-bits.h @@ -25,13 +25,13 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, i #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) LONG_L " $4, %[v]\n\t" "bne $4, %[expect], %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) LONG_L " $4, %[v]\n\t" "bne $4, %[expect], %l[error2]\n\t" #endif @@ -86,13 +86,13 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t e #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) LONG_L " $4, %[v]\n\t" "beq $4, %[expectnot], %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) LONG_L " $4, %[v]\n\t" "beq $4, %[expectnot], %l[error2]\n\t" #endif @@ -149,10 +149,10 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) #endif LONG_L " $4, %[v]\n\t" LONG_ADDI " $4, %[count]\n\t" @@ -204,7 +204,7 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t ex #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) LONG_L " $4, %[v]\n\t" "bne $4, %[expect], %l[cmpfail]\n\t" @@ -213,7 +213,7 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t ex "bne $4, %[expect2], %l[cmpfail]\n\t" RSEQ_INJECT_ASM(5) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) LONG_L " $4, %[v]\n\t" "bne $4, %[expect], %l[error2]\n\t" LONG_L " $4, %[v2]\n\t" @@ -283,13 +283,13 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) LONG_L " $4, %[v]\n\t" "bne $4, %[expect], %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) LONG_L " $4, %[v]\n\t" "bne $4, %[expect], %l[error2]\n\t" #endif @@ -360,13 +360,13 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t LONG_S " %[len], %[rseq_scratch2]\n\t" /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) LONG_L " $4, %[v]\n\t" "bne $4, %[expect], 5f\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 6f) LONG_L " $4, %[v]\n\t" "bne $4, %[expect], 7f\n\t" #endif diff --git a/include/rseq/rseq-mips.h b/include/rseq/rseq-mips.h index 0d47e82..41ea3aa 100644 --- a/include/rseq/rseq-mips.h +++ b/include/rseq/rseq-mips.h @@ -119,7 +119,7 @@ do { \ LONG_S " $4, %[" __rseq_str(rseq_cs) "]\n\t" \ __rseq_str(label) ":\n\t" -#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ +#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \ RSEQ_INJECT_ASM(2) \ "lw $4, %[" __rseq_str(current_cpu_id) "]\n\t" \ "bne $4, %[" __rseq_str(cpu_id) "], " __rseq_str(label) "\n\t" diff --git a/include/rseq/rseq-ppc-bits.h b/include/rseq/rseq-ppc-bits.h index bbc0e0a..b6a5903 100644 --- a/include/rseq/rseq-ppc-bits.h +++ b/include/rseq/rseq-ppc-bits.h @@ -26,16 +26,16 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, i /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) + RSEQ_ASM_OP_CBNE(v, expect, %l[cmpfail]) RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) + RSEQ_ASM_OP_CBNE(v, expect, %l[error2]) #endif /* final store */ RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) @@ -91,16 +91,16 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t e /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) /* cmp @v not equal to @expectnot */ - RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) + RSEQ_ASM_OP_CBEQ(v, expectnot, %l[cmpfail]) RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) /* cmp @v not equal to @expectnot */ - RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2]) + RSEQ_ASM_OP_CBEQ(v, expectnot, %l[error2]) #endif /* load the value of @v */ RSEQ_ASM_OP_R_LOAD(v) @@ -161,11 +161,11 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) #ifdef RSEQ_COMPARE_TWICE /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) #endif /* load the value of @v */ RSEQ_ASM_OP_R_LOAD(v) @@ -221,21 +221,21 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t ex /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) + RSEQ_ASM_OP_CBNE(v, expect, %l[cmpfail]) RSEQ_INJECT_ASM(4) /* cmp @v2 equal to @expct2 */ - RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) + RSEQ_ASM_OP_CBNE(v2, expect2, %l[cmpfail]) RSEQ_INJECT_ASM(5) #ifdef RSEQ_COMPARE_TWICE /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) + RSEQ_ASM_OP_CBNE(v, expect, %l[error2]) /* cmp @v2 equal to @expct2 */ - RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3]) + RSEQ_ASM_OP_CBNE(v2, expect2, %l[error3]) #endif /* final store */ RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) @@ -305,16 +305,16 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) + RSEQ_ASM_OP_CBNE(v, expect, %l[cmpfail]) RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) + RSEQ_ASM_OP_CBNE(v, expect, %l[error2]) #endif /* try store */ RSEQ_ASM_OP_STORE(newv2, v2) @@ -386,16 +386,16 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) + RSEQ_ASM_OP_CBNE(v, expect, %l[cmpfail]) RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE /* cmp cpuid */ - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) /* cmp @v equal to @expect */ - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) + RSEQ_ASM_OP_CBNE(v, expect, %l[error2]) #endif /* try memcpy */ RSEQ_ASM_OP_R_MEMCPY() diff --git a/include/rseq/rseq-ppc.h b/include/rseq/rseq-ppc.h index ac39dfd..9105e21 100644 --- a/include/rseq/rseq-ppc.h +++ b/include/rseq/rseq-ppc.h @@ -136,7 +136,7 @@ do { \ __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \ (post_commit_ip - start_ip), abort_ip) -#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ +#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \ RSEQ_INJECT_ASM(2) \ RSEQ_LOAD_INT(current_cpu_id) "%%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \ "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t" \ @@ -154,12 +154,12 @@ do { \ * RSEQ_ASM_OP_R_*: has hard-code registers in it * RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7) */ -#define RSEQ_ASM_OP_CMPEQ(var, expect, label) \ +#define RSEQ_ASM_OP_CBNE(var, expect, label) \ RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \ RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \ "bne- cr7, " __rseq_str(label) "\n\t" -#define RSEQ_ASM_OP_CMPNE(var, expectnot, label) \ +#define RSEQ_ASM_OP_CBEQ(var, expectnot, label) \ RSEQ_LOAD_LONG(var) "%%r17, %[" __rseq_str(var) "]\n\t" \ RSEQ_CMP_LONG "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \ "beq- cr7, " __rseq_str(label) "\n\t" diff --git a/include/rseq/rseq-riscv-bits.h b/include/rseq/rseq-riscv-bits.h index 578b7de..83cd637 100644 --- a/include/rseq/rseq-riscv-bits.h +++ b/include/rseq/rseq-riscv-bits.h @@ -18,13 +18,13 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, i RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") #endif RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") + RSEQ_ASM_OP_CBNE(v, expect, "%l[cmpfail]") RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") - RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") + RSEQ_ASM_OP_CBNE(v, expect, "%l[error2]") #endif RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) RSEQ_INJECT_ASM(5) @@ -72,13 +72,13 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t e RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") #endif RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[cmpfail]") + RSEQ_ASM_OP_CBEQ(v, expectnot, "%l[cmpfail]") RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") - RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[error2]") + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") + RSEQ_ASM_OP_CBEQ(v, expectnot, "%l[error2]") #endif RSEQ_ASM_OP_R_LOAD(v) RSEQ_ASM_OP_R_STORE(load) @@ -126,10 +126,10 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") #endif RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") #endif RSEQ_ASM_OP_R_LOAD(v) RSEQ_ASM_OP_R_ADD(count) @@ -175,16 +175,16 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t ex RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error3]") #endif RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") + RSEQ_ASM_OP_CBNE(v, expect, "%l[cmpfail]") RSEQ_INJECT_ASM(4) - RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[cmpfail]") + RSEQ_ASM_OP_CBNE(v2, expect2, "%l[cmpfail]") RSEQ_INJECT_ASM(5) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") - RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") - RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[error3]") + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") + RSEQ_ASM_OP_CBNE(v, expect, "%l[error2]") + RSEQ_ASM_OP_CBNE(v2, expect2, "%l[error3]") #endif RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) RSEQ_INJECT_ASM(6) @@ -239,10 +239,10 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, off_t off, i RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]") #endif RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") #endif RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, 3) RSEQ_INJECT_ASM(4) @@ -292,13 +292,13 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") #endif RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") + RSEQ_ASM_OP_CBNE(v, expect, "%l[cmpfail]") RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") - RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") + RSEQ_ASM_OP_CBNE(v, expect, "%l[error2]") #endif RSEQ_ASM_OP_STORE(newv2, v2) RSEQ_INJECT_ASM(5) @@ -354,13 +354,13 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]") #endif RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) - RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]") + RSEQ_ASM_OP_CBNE(v, expect, "%l[cmpfail]") RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") - RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]") + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, "%l[error1]") + RSEQ_ASM_OP_CBNE(v, expect, "%l[error2]") #endif RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) RSEQ_INJECT_ASM(5) diff --git a/include/rseq/rseq-riscv.h b/include/rseq/rseq-riscv.h index 193995a..3047b6e 100644 --- a/include/rseq/rseq-riscv.h +++ b/include/rseq/rseq-riscv.h @@ -99,24 +99,24 @@ do { \ #define RSEQ_ASM_OP_STORE(value, var) \ REG_S "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n" -#define RSEQ_ASM_OP_CMPEQ(var, expect, label) \ +#define RSEQ_ASM_OP_CBNE(var, expect, label) \ REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \ "bne " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \ __rseq_str(label) "\n" -#define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \ +#define RSEQ_ASM_OP_CBNE32(var, expect, label) \ "lw " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \ "bne " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \ __rseq_str(label) "\n" -#define RSEQ_ASM_OP_CMPNE(var, expect, label) \ +#define RSEQ_ASM_OP_CBEQ(var, expect, label) \ REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \ "beq " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \ __rseq_str(label) "\n" -#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ +#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \ RSEQ_INJECT_ASM(2) \ - RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label) + RSEQ_ASM_OP_CBNE32(current_cpu_id, cpu_id, label) #define RSEQ_ASM_OP_R_LOAD(var) \ REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" diff --git a/include/rseq/rseq-s390-bits.h b/include/rseq/rseq-s390-bits.h index fe150ce..c3641b7 100644 --- a/include/rseq/rseq-s390-bits.h +++ b/include/rseq/rseq-s390-bits.h @@ -20,13 +20,13 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, i #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) LONG_CMP " %[expect], %[v]\n\t" "jnz %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) LONG_CMP " %[expect], %[v]\n\t" "jnz %l[error2]\n\t" #endif @@ -88,14 +88,14 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t e #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) LONG_L " %%r1, %[v]\n\t" LONG_CMP_R " %%r1, %[expectnot]\n\t" "je %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) LONG_L " %%r1, %[v]\n\t" LONG_CMP_R " %%r1, %[expectnot]\n\t" "je %l[error2]\n\t" @@ -156,10 +156,10 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) #endif LONG_L " %%r0, %[v]\n\t" LONG_ADD_R " %%r0, %[count]\n\t" @@ -213,7 +213,7 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t ex #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) LONG_CMP " %[expect], %[v]\n\t" "jnz %l[cmpfail]\n\t" @@ -222,7 +222,7 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t ex "jnz %l[cmpfail]\n\t" RSEQ_INJECT_ASM(5) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) LONG_CMP " %[expect], %[v]\n\t" "jnz %l[error2]\n\t" LONG_CMP " %[expect2], %[v2]\n\t" @@ -297,13 +297,13 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) LONG_CMP " %[expect], %[v]\n\t" "jnz %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) LONG_CMP " %[expect], %[v]\n\t" "jnz %l[error2]\n\t" #endif @@ -375,13 +375,13 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t LONG_S " %[len], %[rseq_scratch2]\n\t" /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) LONG_CMP " %[expect], %[v]\n\t" "jnz 5f\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 6f) LONG_CMP " %[expect], %[v]\n\t" "jnz 7f\n\t" #endif diff --git a/include/rseq/rseq-s390.h b/include/rseq/rseq-s390.h index 76ce74f..8158826 100644 --- a/include/rseq/rseq-s390.h +++ b/include/rseq/rseq-s390.h @@ -111,7 +111,7 @@ do { \ LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t" \ __rseq_str(label) ":\n\t" -#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ +#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \ RSEQ_INJECT_ASM(2) \ "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \ "jnz " __rseq_str(label) "\n\t" diff --git a/include/rseq/rseq-x86-bits.h b/include/rseq/rseq-x86-bits.h index 8ee71cb..fd3c6b8 100644 --- a/include/rseq/rseq-x86-bits.h +++ b/include/rseq/rseq-x86-bits.h @@ -25,13 +25,13 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, i #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) "cmpq %[v], %[expect]\n\t" "jnz %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) "cmpq %[v], %[expect]\n\t" "jnz %l[error2]\n\t" #endif @@ -91,14 +91,14 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t e #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) "movq %[v], %%rbx\n\t" "cmpq %%rbx, %[expectnot]\n\t" "je %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) "movq %[v], %%rbx\n\t" "cmpq %%rbx, %[expectnot]\n\t" "je %l[error2]\n\t" @@ -157,10 +157,10 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) #endif /* final store */ "addq %[count], %[v]\n\t" @@ -211,10 +211,10 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, long off, in #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) #endif /* get p+v */ "movq %[ptr], %%rbx\n\t" @@ -267,7 +267,7 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t ex #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) "cmpq %[v], %[expect]\n\t" "jnz %l[cmpfail]\n\t" @@ -276,7 +276,7 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t ex "jnz %l[cmpfail]\n\t" RSEQ_INJECT_ASM(5) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) "cmpq %[v], %[expect]\n\t" "jnz %l[error2]\n\t" "cmpq %[v2], %[expect2]\n\t" @@ -348,13 +348,13 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) "cmpq %[v], %[expect]\n\t" "jnz %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) "cmpq %[v], %[expect]\n\t" "jnz %l[error2]\n\t" #endif @@ -423,13 +423,13 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t "movq %[len], %[rseq_scratch2]\n\t" /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) "cmpq %[v], %[expect]\n\t" "jnz 5f\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f) "cmpq %[v], %[expect]\n\t" "jnz 7f\n\t" #endif @@ -537,13 +537,13 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, i #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) "cmpl %[v], %[expect]\n\t" "jnz %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) "cmpl %[v], %[expect]\n\t" "jnz %l[error2]\n\t" #endif @@ -603,14 +603,14 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t e #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) "movl %[v], %%ebx\n\t" "cmpl %%ebx, %[expectnot]\n\t" "je %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) "movl %[v], %%ebx\n\t" "cmpl %%ebx, %[expectnot]\n\t" "je %l[error2]\n\t" @@ -669,10 +669,10 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) #endif /* final store */ "addl %[count], %[v]\n\t" @@ -722,7 +722,7 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t ex #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) "cmpl %[v], %[expect]\n\t" "jnz %l[cmpfail]\n\t" @@ -731,7 +731,7 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t ex "jnz %l[cmpfail]\n\t" RSEQ_INJECT_ASM(5) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) "cmpl %[v], %[expect]\n\t" "jnz %l[error2]\n\t" "cmpl %[expect2], %[v2]\n\t" @@ -804,14 +804,14 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t #endif /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) "movl %[expect], %%eax\n\t" "cmpl %[v], %%eax\n\t" "jnz %l[cmpfail]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), %l[error1]) "movl %[expect], %%eax\n\t" "cmpl %[v], %%eax\n\t" "jnz %l[error2]\n\t" @@ -886,14 +886,14 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t "movl %[len], %[rseq_scratch2]\n\t" /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset])) - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 4f) RSEQ_INJECT_ASM(3) "movl %[expect], %%eax\n\t" "cmpl %%eax, %[v]\n\t" "jnz 5f\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_TEMPLATE_CPU_ID_OFFSET(%[rseq_offset]), 6f) "movl %[expect], %%eax\n\t" "cmpl %%eax, %[v]\n\t" "jnz 7f\n\t" diff --git a/include/rseq/rseq-x86.h b/include/rseq/rseq-x86.h index 9672cb8..ecd7560 100644 --- a/include/rseq/rseq-x86.h +++ b/include/rseq/rseq-x86.h @@ -91,7 +91,7 @@ do { \ "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \ __rseq_str(label) ":\n\t" -#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ +#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \ RSEQ_INJECT_ASM(2) \ "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ "jnz " __rseq_str(label) "\n\t" @@ -177,7 +177,7 @@ do { \ "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \ __rseq_str(label) ":\n\t" -#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \ +#define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label) \ RSEQ_INJECT_ASM(2) \ "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \ "jnz " __rseq_str(label) "\n\t"