x86-32: use %gs segment selector for accessing rseq thread area

Rather than use rseq_get_abi() and pass its result through a register to
the inline assembler, directly access the per-thread rseq area through a
memory reference combining the %gs segment selector, the constant offset
of the field in struct rseq, and the rseq_offset value (in a register).

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Change-Id: Ibd6fac4f2f80774ab74ae495cd0f688d789f36b8

diff --git a/include/rseq/rseq-x86.h b/include/rseq/rseq-x86.h
index d39f9c03cfec8ec5c3d4092060f63bb324c8dbb8..944a13cbea02004dd7fc43f609b4e72c69e46e5e 100644 (file)
--- a/include/rseq/rseq-x86.h
+++ b/include/rseq/rseq-x86.h
@@ -693,6 +693,8 @@ error1:
 
 #elif defined(__i386__)
 
+#define RSEQ_ASM_TP_SEGMENT    %%gs
+
 #define rseq_smp_mb()  \
        __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
 #define rseq_smp_rmb() \
@@ -792,14 +794,14 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 #endif
                /* Start rseq by storing table entry pointer into rseq_cs. */
-               RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
-               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
+               RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
+               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
                RSEQ_INJECT_ASM(3)
                "cmpl %[v], %[expect]\n\t"
                "jnz %l[cmpfail]\n\t"
                RSEQ_INJECT_ASM(4)
 #ifdef RSEQ_COMPARE_TWICE
-               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
+               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
                "cmpl %[v], %[expect]\n\t"
                "jnz %l[error2]\n\t"
 #endif
@@ -810,7 +812,7 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
                : /* gcc asm goto does not allow outputs */
                : [cpu_id]              "r" (cpu),
-                 [rseq_abi]            "r" (rseq_get_abi()),
+                 [rseq_offset]         "r" (rseq_offset),
                  [v]                   "m" (*v),
                  [expect]              "r" (expect),
                  [newv]                "r" (newv)
@@ -858,15 +860,15 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 #endif
                /* Start rseq by storing table entry pointer into rseq_cs. */
-               RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
-               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
+               RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
+               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
                RSEQ_INJECT_ASM(3)
                "movl %[v], %%ebx\n\t"
                "cmpl %%ebx, %[expectnot]\n\t"
                "je %l[cmpfail]\n\t"
                RSEQ_INJECT_ASM(4)
 #ifdef RSEQ_COMPARE_TWICE
-               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
+               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
                "movl %[v], %%ebx\n\t"
                "cmpl %%ebx, %[expectnot]\n\t"
                "je %l[error2]\n\t"
@@ -881,7 +883,7 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
                : /* gcc asm goto does not allow outputs */
                : [cpu_id]              "r" (cpu),
-                 [rseq_abi]            "r" (rseq_get_abi()),
+                 [rseq_offset]         "r" (rseq_offset),
                  /* final store input */
                  [v]                   "m" (*v),
                  [expectnot]           "r" (expectnot),
@@ -924,11 +926,11 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 #endif
                /* Start rseq by storing table entry pointer into rseq_cs. */
-               RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
-               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
+               RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
+               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
                RSEQ_INJECT_ASM(3)
 #ifdef RSEQ_COMPARE_TWICE
-               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
+               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
 #endif
                /* final store */
                "addl %[count], %[v]\n\t"
@@ -937,7 +939,7 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
                : /* gcc asm goto does not allow outputs */
                : [cpu_id]              "r" (cpu),
-                 [rseq_abi]            "r" (rseq_get_abi()),
+                 [rseq_offset]         "r" (rseq_offset),
                  /* final store input */
                  [v]                   "m" (*v),
                  [count]               "ir" (count)
@@ -976,14 +978,14 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 #endif
                /* Start rseq by storing table entry pointer into rseq_cs. */
-               RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
-               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
+               RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
+               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
                RSEQ_INJECT_ASM(3)
                "cmpl %[v], %[expect]\n\t"
                "jnz %l[cmpfail]\n\t"
                RSEQ_INJECT_ASM(4)
 #ifdef RSEQ_COMPARE_TWICE
-               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
+               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
                "cmpl %[v], %[expect]\n\t"
                "jnz %l[error2]\n\t"
 #endif
@@ -998,7 +1000,7 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
                : /* gcc asm goto does not allow outputs */
                : [cpu_id]              "r" (cpu),
-                 [rseq_abi]            "r" (rseq_get_abi()),
+                 [rseq_offset]         "r" (rseq_offset),
                  /* try store input */
                  [v2]                  "m" (*v2),
                  [newv2]               "m" (newv2),
@@ -1047,15 +1049,15 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 #endif
                /* Start rseq by storing table entry pointer into rseq_cs. */
-               RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
-               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
+               RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
+               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
                RSEQ_INJECT_ASM(3)
                "movl %[expect], %%eax\n\t"
                "cmpl %[v], %%eax\n\t"
                "jnz %l[cmpfail]\n\t"
                RSEQ_INJECT_ASM(4)
 #ifdef RSEQ_COMPARE_TWICE
-               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
+               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
                "movl %[expect], %%eax\n\t"
                "cmpl %[v], %%eax\n\t"
                "jnz %l[error2]\n\t"
@@ -1071,7 +1073,7 @@ int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
                : /* gcc asm goto does not allow outputs */
                : [cpu_id]              "r" (cpu),
-                 [rseq_abi]            "r" (rseq_get_abi()),
+                 [rseq_offset]         "r" (rseq_offset),
                  /* try store input */
                  [v2]                  "m" (*v2),
                  [newv2]               "r" (newv2),
@@ -1122,8 +1124,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
 #endif
                /* Start rseq by storing table entry pointer into rseq_cs. */
-               RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
-               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
+               RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
+               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
                RSEQ_INJECT_ASM(3)
                "cmpl %[v], %[expect]\n\t"
                "jnz %l[cmpfail]\n\t"
@@ -1132,7 +1134,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
                "jnz %l[cmpfail]\n\t"
                RSEQ_INJECT_ASM(5)
 #ifdef RSEQ_COMPARE_TWICE
-               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
+               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
                "cmpl %[v], %[expect]\n\t"
                "jnz %l[error2]\n\t"
                "cmpl %[expect2], %[v2]\n\t"
@@ -1146,7 +1148,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
                : /* gcc asm goto does not allow outputs */
                : [cpu_id]              "r" (cpu),
-                 [rseq_abi]            "r" (rseq_get_abi()),
+                 [rseq_offset]         "r" (rseq_offset),
                  /* cmp2 input */
                  [v2]                  "m" (*v2),
                  [expect2]             "r" (expect2),
@@ -1204,15 +1206,15 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
                "movl %[dst], %[rseq_scratch1]\n\t"
                "movl %[len], %[rseq_scratch2]\n\t"
                /* Start rseq by storing table entry pointer into rseq_cs. */
-               RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
-               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
+               RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
+               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
                RSEQ_INJECT_ASM(3)
                "movl %[expect], %%eax\n\t"
                "cmpl %%eax, %[v]\n\t"
                "jnz 5f\n\t"
                RSEQ_INJECT_ASM(4)
 #ifdef RSEQ_COMPARE_TWICE
-               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
+               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
                "movl %[expect], %%eax\n\t"
                "cmpl %%eax, %[v]\n\t"
                "jnz 7f\n\t"
@@ -1262,7 +1264,7 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 #endif
                : /* gcc asm goto does not allow outputs */
                : [cpu_id]              "r" (cpu),
-                 [rseq_abi]            "r" (rseq_get_abi()),
+                 [rseq_offset]         "r" (rseq_offset),
                  /* final store input */
                  [v]                   "m" (*v),
                  [expect]              "m" (expect),
@@ -1321,15 +1323,15 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
                "movl %[dst], %[rseq_scratch1]\n\t"
                "movl %[len], %[rseq_scratch2]\n\t"
                /* Start rseq by storing table entry pointer into rseq_cs. */
-               RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
-               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
+               RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
+               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
                RSEQ_INJECT_ASM(3)
                "movl %[expect], %%eax\n\t"
                "cmpl %%eax, %[v]\n\t"
                "jnz 5f\n\t"
                RSEQ_INJECT_ASM(4)
 #ifdef RSEQ_COMPARE_TWICE
-               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
+               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
                "movl %[expect], %%eax\n\t"
                "cmpl %%eax, %[v]\n\t"
                "jnz 7f\n\t"
@@ -1380,7 +1382,7 @@ int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
 #endif
                : /* gcc asm goto does not allow outputs */
                : [cpu_id]              "r" (cpu),
-                 [rseq_abi]            "r" (rseq_get_abi()),
+                 [rseq_offset]         "r" (rseq_offset),
                  /* final store input */
                  [v]                   "m" (*v),
                  [expect]              "m" (expect),
@@ -1433,13 +1435,13 @@ int rseq_deref_loadoffp(intptr_t *p, long voffp, intptr_t *load, int cpu)
                RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 #endif
                /* Start rseq by storing table entry pointer into rseq_cs. */
-               RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
-               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
+               RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
+               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
                RSEQ_INJECT_ASM(3)
                "movl %[p], %%ebx\n\t"
                RSEQ_INJECT_ASM(4)
 #ifdef RSEQ_COMPARE_TWICE
-               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
+               RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
 #endif
                "addl %[voffp], %%ebx\n\t"
                "movl (%%ebx), %%ebx\n\t"
@@ -1449,7 +1451,7 @@ int rseq_deref_loadoffp(intptr_t *p, long voffp, intptr_t *load, int cpu)
                RSEQ_ASM_DEFINE_ABORT(4, "", abort)
                : /* gcc asm goto does not allow outputs */
                : [cpu_id]              "r" (cpu),
-                 [rseq_abi]            "r" (rseq_get_abi()),
+                 [rseq_offset]         "r" (rseq_offset),
                  /* final store input */
                  [p]                   "m" (*p),
                  [voffp]               "ir" (voffp),