From f9672d409834b7ee148495fd822e81d180bcea4c Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Sat, 2 Mar 2024 11:08:25 -0500 Subject: [PATCH] arm: work-around register pressure limitations The recent move to percpu allocator breaks the 32-bit ARM build due to register pressure. Fix this by using explicit register clobbers and loading input values from "m" input operands. Note that old gcc does not support output operands for asm goto, so we cannot simply re-use an input register as output. Signed-off-by: Mathieu Desnoyers Change-Id: I1d546e427c7f6b62035433815806c8ec275a55f8 --- include/rseq/arch/arm/bits.h | 111 ++++++++++++++++------------------- 1 file changed, 50 insertions(+), 61 deletions(-) diff --git a/include/rseq/arch/arm/bits.h b/include/rseq/arch/arm/bits.h index b5b99f8..b41da94 100644 --- a/include/rseq/arch/arm/bits.h +++ b/include/rseq/arch/arm/bits.h @@ -384,7 +384,21 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_memcpy_store__ptr)(intptr_t *v, intp void *dst, void *src, size_t len, intptr_t newv, int cpu) { - uint32_t rseq_scratch[3]; + /* + * Work-around register pressure limitations. + * Old gcc does not support output operands for asm goto, so + * input registers cannot simply be re-used as output registers. + * This is why clobbered registers are used. + */ + struct rseq_local { + uint32_t expect, dst, src, len, newv; + } rseq_local = { + .expect = (uint32_t) expect, + .dst = (uint32_t) dst, + .src = (uint32_t) src, + .len = (uint32_t) len, + .newv = (uint32_t) newv, + }; RSEQ_INJECT_C(9) @@ -395,91 +409,66 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_memcpy_store__ptr)(intptr_t *v, intp RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) #endif - "str %[src], %[rseq_scratch0]\n\t" - "str %[dst], %[rseq_scratch1]\n\t" - "str %[len], %[rseq_scratch2]\n\t" /* Start rseq by storing table entry pointer into rseq_cs. */ RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f) RSEQ_INJECT_ASM(3) "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne 5f\n\t" + /* load expect into r5 */ + "ldr r5, %[expect]\n\t" + "cmp r5, r0\n\t" + "bne %l[ne]\n\t" RSEQ_INJECT_ASM(4) #ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 6f) + RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1]) "ldr r0, %[v]\n\t" - "cmp %[expect], r0\n\t" - "bne 7f\n\t" + "cmp r5, r0\n\t" + "bne %l[error2]\n\t" #endif + /* load dst into r5 */ + "ldr r5, %[dst]\n\t" + /* load src into r6 */ + "ldr r6, %[src]\n\t" + /* load len into r7 */ + "ldr r7, %[len]\n\t" /* try memcpy */ - "cmp %[len], #0\n\t" \ - "beq 333f\n\t" \ - "222:\n\t" \ - "ldrb %%r0, [%[src]]\n\t" \ - "strb %%r0, [%[dst]]\n\t" \ - "adds %[src], #1\n\t" \ - "adds %[dst], #1\n\t" \ - "subs %[len], #1\n\t" \ - "bne 222b\n\t" \ - "333:\n\t" \ + "cmp r7, #0\n\t" + "beq 333f\n\t" + "222:\n\t" + "ldrb %%r0, [r6]\n\t" + "strb %%r0, [r5]\n\t" + "adds r6, #1\n\t" + "adds r5, #1\n\t" + "subs r7, #1\n\t" + "bne 222b\n\t" + "333:\n\t" RSEQ_INJECT_ASM(5) #ifdef RSEQ_TEMPLATE_MO_RELEASE "dmb\n\t" /* full mb provides store-release */ #endif + /* load newv into r5 */ + "ldr r5, %[newv]\n\t" /* final store */ - "str %[newv], %[v]\n\t" + "str r5, %[v]\n\t" "2:\n\t" RSEQ_INJECT_ASM(6) - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t" - "b 8f\n\t" - RSEQ_ASM_DEFINE_ABORT(4, - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t", - abort, 3, 1b, 2b, 4f) - RSEQ_ASM_DEFINE_TEARDOWN(5, - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t", - ne) -#ifdef RSEQ_COMPARE_TWICE - RSEQ_ASM_DEFINE_TEARDOWN(6, - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t", - error1) - RSEQ_ASM_DEFINE_TEARDOWN(7, - /* teardown */ - "ldr %[len], %[rseq_scratch2]\n\t" - "ldr %[dst], %[rseq_scratch1]\n\t" - "ldr %[src], %[rseq_scratch0]\n\t", - error2) -#endif - "8:\n\t" + "b 5f\n\t" + RSEQ_ASM_DEFINE_ABORT(4, "", abort, 3, 1b, 2b, 4f) + "5:\n\t" : /* gcc asm goto does not allow outputs */ : [cpu_id] "r" (cpu), [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_INDEX_CPU_ID_FIELD), [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), /* final store input */ [v] "m" (*v), - [expect] "r" (expect), - [newv] "r" (newv), /* try memcpy input */ - [dst] "r" (dst), - [src] "r" (src), - [len] "r" (len), - [rseq_scratch0] "m" (rseq_scratch[0]), - [rseq_scratch1] "m" (rseq_scratch[1]), - [rseq_scratch2] "m" (rseq_scratch[2]) + [expect] "m" (rseq_local.expect), /* r5 */ + [dst] "m" (rseq_local.dst), /* r5 */ + [src] "m" (rseq_local.src), /* r6 */ + [len] "m" (rseq_local.len), /* r7 */ + [newv] "m" (rseq_local.newv) /* r5 */ RSEQ_INJECT_INPUT - : "r0", "memory", "cc" + : "r0", "r5", "r6", "r7", "memory", "cc" RSEQ_INJECT_CLOBBER : abort, ne #ifdef RSEQ_COMPARE_TWICE -- 2.34.1