arm: work-around register pressure limitations

The recent move to percpu allocator breaks the 32-bit ARM build due to
register pressure. Fix this by using explicit register clobbers and
loading input values from "m" input operands.

Note that old gcc does not support output operands for asm goto, so we
cannot simply re-use an input register as output.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Change-Id: I1d546e427c7f6b62035433815806c8ec275a55f8

diff --git a/include/rseq/arch/arm/bits.h b/include/rseq/arch/arm/bits.h
index b5b99f8e71e869f18b5c8ee483e411a6ff95e1b0..b41da94812b70d79c6993fe2f8313849d26953e7 100644 (file)
--- a/include/rseq/arch/arm/bits.h
+++ b/include/rseq/arch/arm/bits.h
@@ -384,7 +384,21 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_memcpy_store__ptr)(intptr_t *v, intp
                                 void *dst, void *src, size_t len,
                                 intptr_t newv, int cpu)
 {
-       uint32_t rseq_scratch[3];
+       /*
+        * Work-around register pressure limitations.
+        * Old gcc does not support output operands for asm goto, so
+        * input registers cannot simply be re-used as output registers.
+        * This is why clobbered registers are used.
+        */
+       struct rseq_local {
+               uint32_t expect, dst, src, len, newv;
+       } rseq_local = {
+               .expect = (uint32_t) expect,
+               .dst = (uint32_t) dst,
+               .src = (uint32_t) src,
+               .len = (uint32_t) len,
+               .newv = (uint32_t) newv,
+       };
 
        RSEQ_INJECT_C(9)
 
@@ -395,91 +409,66 @@ int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_memcpy_store__ptr)(intptr_t *v, intp
                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 #endif
-               "str %[src], %[rseq_scratch0]\n\t"
-               "str %[dst], %[rseq_scratch1]\n\t"
-               "str %[len], %[rseq_scratch2]\n\t"
                /* Start rseq by storing table entry pointer into rseq_cs. */
                RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
                RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
                RSEQ_INJECT_ASM(3)
                "ldr r0, %[v]\n\t"
-               "cmp %[expect], r0\n\t"
-               "bne 5f\n\t"
+               /* load expect into r5 */
+               "ldr r5, %[expect]\n\t"
+               "cmp r5, r0\n\t"
+               "bne %l[ne]\n\t"
                RSEQ_INJECT_ASM(4)
 #ifdef RSEQ_COMPARE_TWICE
-               RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 6f)
+               RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
                "ldr r0, %[v]\n\t"
-               "cmp %[expect], r0\n\t"
-               "bne 7f\n\t"
+               "cmp r5, r0\n\t"
+               "bne %l[error2]\n\t"
 #endif
+               /* load dst into r5 */
+               "ldr r5, %[dst]\n\t"
+               /* load src into r6 */
+               "ldr r6, %[src]\n\t"
+               /* load len into r7 */
+               "ldr r7, %[len]\n\t"
                /* try memcpy */
-               "cmp %[len], #0\n\t" \
-               "beq 333f\n\t" \
-               "222:\n\t" \
-               "ldrb %%r0, [%[src]]\n\t" \
-               "strb %%r0, [%[dst]]\n\t" \
-               "adds %[src], #1\n\t" \
-               "adds %[dst], #1\n\t" \
-               "subs %[len], #1\n\t" \
-               "bne 222b\n\t" \
-               "333:\n\t" \
+               "cmp r7, #0\n\t"
+               "beq 333f\n\t"
+               "222:\n\t"
+               "ldrb %%r0, [r6]\n\t"
+               "strb %%r0, [r5]\n\t"
+               "adds r6, #1\n\t"
+               "adds r5, #1\n\t"
+               "subs r7, #1\n\t"
+               "bne 222b\n\t"
+               "333:\n\t"
                RSEQ_INJECT_ASM(5)
 #ifdef RSEQ_TEMPLATE_MO_RELEASE
                "dmb\n\t"       /* full mb provides store-release */
 #endif
+               /* load newv into r5 */
+               "ldr r5, %[newv]\n\t"
                /* final store */
-               "str %[newv], %[v]\n\t"
+               "str r5, %[v]\n\t"
                "2:\n\t"
                RSEQ_INJECT_ASM(6)
-               /* teardown */
-               "ldr %[len], %[rseq_scratch2]\n\t"
-               "ldr %[dst], %[rseq_scratch1]\n\t"
-               "ldr %[src], %[rseq_scratch0]\n\t"
-               "b 8f\n\t"
-               RSEQ_ASM_DEFINE_ABORT(4,
-                                     /* teardown */
-                                     "ldr %[len], %[rseq_scratch2]\n\t"
-                                     "ldr %[dst], %[rseq_scratch1]\n\t"
-                                     "ldr %[src], %[rseq_scratch0]\n\t",
-                                     abort, 3, 1b, 2b, 4f)
-               RSEQ_ASM_DEFINE_TEARDOWN(5,
-                                       /* teardown */
-                                       "ldr %[len], %[rseq_scratch2]\n\t"
-                                       "ldr %[dst], %[rseq_scratch1]\n\t"
-                                       "ldr %[src], %[rseq_scratch0]\n\t",
-                                       ne)
-#ifdef RSEQ_COMPARE_TWICE
-               RSEQ_ASM_DEFINE_TEARDOWN(6,
-                                       /* teardown */
-                                       "ldr %[len], %[rseq_scratch2]\n\t"
-                                       "ldr %[dst], %[rseq_scratch1]\n\t"
-                                       "ldr %[src], %[rseq_scratch0]\n\t",
-                                       error1)
-               RSEQ_ASM_DEFINE_TEARDOWN(7,
-                                       /* teardown */
-                                       "ldr %[len], %[rseq_scratch2]\n\t"
-                                       "ldr %[dst], %[rseq_scratch1]\n\t"
-                                       "ldr %[src], %[rseq_scratch0]\n\t",
-                                       error2)
-#endif
-               "8:\n\t"
+               "b 5f\n\t"
+               RSEQ_ASM_DEFINE_ABORT(4, "", abort, 3, 1b, 2b, 4f)
+               "5:\n\t"
                : /* gcc asm goto does not allow outputs */
                : [cpu_id]              "r" (cpu),
                  [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_INDEX_CPU_ID_FIELD),
                  [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
                  /* final store input */
                  [v]                   "m" (*v),
-                 [expect]              "r" (expect),
-                 [newv]                "r" (newv),
                  /* try memcpy input */
-                 [dst]                 "r" (dst),
-                 [src]                 "r" (src),
-                 [len]                 "r" (len),
-                 [rseq_scratch0]       "m" (rseq_scratch[0]),
-                 [rseq_scratch1]       "m" (rseq_scratch[1]),
-                 [rseq_scratch2]       "m" (rseq_scratch[2])
+                 [expect]              "m" (rseq_local.expect),        /* r5 */
+                 [dst]                 "m" (rseq_local.dst),           /* r5 */
+                 [src]                 "m" (rseq_local.src),           /* r6 */
+                 [len]                 "m" (rseq_local.len),           /* r7 */
+                 [newv]                "m" (rseq_local.newv)           /* r5 */
                  RSEQ_INJECT_INPUT
-               : "r0", "memory", "cc"
+               : "r0", "r5", "r6", "r7", "memory", "cc"
                  RSEQ_INJECT_CLOBBER
                : abort, ne
 #ifdef RSEQ_COMPARE_TWICE