include/rseq/rseq-s390.h

   1 /* SPDX-License-Identifier: LGPL-2.1-only OR MIT */
   2
   3 /*
   4  * RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the
   5  * access-register mode nor the linkage stack this instruction will always
   6  * cause a special-operation exception (the trap-enabled bit in the DUCT
   7  * is and will stay 0). The instruction pattern is
   8  *      b2 ff 0f ff     trap4   4095(%r0)
   9  */
  10 #define RSEQ_SIG        0xB2FF0FFF
  11
  12 #define rseq_smp_mb()   __asm__ __volatile__ ("bcr 15,0" ::: "memory")
  13 #define rseq_smp_rmb()  rseq_smp_mb()
  14 #define rseq_smp_wmb()  rseq_smp_mb()
  15
  16 #define rseq_smp_load_acquire(p)                                        \
  17 __extension__ ({                                                        \
  18         __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
  19         rseq_barrier();                                                 \
  20         ____p1;                                                         \
  21 })
  22
  23 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  24
  25 #define rseq_smp_store_release(p, v)                                    \
  26 do {                                                                    \
  27         rseq_barrier();                                                 \
  28         RSEQ_WRITE_ONCE(*p, v);                                         \
  29 } while (0)
  30
  31 #ifdef RSEQ_SKIP_FASTPATH
  32 #include "rseq-skip.h"
  33 #else /* !RSEQ_SKIP_FASTPATH */
  34
  35 #ifdef __s390x__
  36
  37 #define RSEQ_CS_PTR             rseq_cs.ptr
  38
  39 #define LONG_L                  "lg"
  40 #define LONG_S                  "stg"
  41 #define LONG_LT_R               "ltgr"
  42 #define LONG_CMP                "cg"
  43 #define LONG_CMP_R              "cgr"
  44 #define LONG_ADDI               "aghi"
  45 #define LONG_ADD_R              "agr"
  46
  47 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
  48                                 start_ip, post_commit_offset, abort_ip) \
  49                 ".pushsection __rseq_cs, \"aw\"\n\t"                    \
  50                 ".balign 32\n\t"                                        \
  51                 __rseq_str(label) ":\n\t"                               \
  52                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  53                 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
  54                 ".popsection\n\t"                                       \
  55                 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
  56                 ".quad " __rseq_str(label) "b\n\t"                      \
  57                 ".popsection\n\t"
  58
  59 /*
  60  * Exit points of a rseq critical section consist of all instructions outside
  61  * of the critical section where a critical section can either branch to or
  62  * reach through the normal course of its execution. The abort IP and the
  63  * post-commit IP are already part of the __rseq_cs section and should not be
  64  * explicitly defined as additional exit points. Knowing all exit points is
  65  * useful to assist debuggers stepping over the critical section.
  66  */
  67 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
  68                 ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
  69                 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
  70                 ".popsection\n\t"
  71
  72 #elif __s390__
  73
  74 #define RSEQ_CS_PTR             rseq_cs.ptr.ptr32
  75
  76 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
  77                                 start_ip, post_commit_offset, abort_ip) \
  78                 ".pushsection __rseq_cs, \"aw\"\n\t"                    \
  79                 ".balign 32\n\t"                                        \
  80                 __rseq_str(label) ":\n\t"                               \
  81                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  82                 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
  83                 ".popsection\n\t"                                       \
  84                 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
  85                 ".long 0x0, " __rseq_str(label) "b\n\t"                 \
  86                 ".popsection\n\t"
  87
  88 /*
  89  * Exit points of a rseq critical section consist of all instructions outside
  90  * of the critical section where a critical section can either branch to or
  91  * reach through the normal course of its execution. The abort IP and the
  92  * post-commit IP are already part of the __rseq_cs section and should not be
  93  * explicitly defined as additional exit points. Knowing all exit points is
  94  * useful to assist debuggers stepping over the critical section.
  95  */
  96 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
  97                 ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
  98                 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
  99                 ".popsection\n\t"
 100
 101 #define LONG_L                  "l"
 102 #define LONG_S                  "st"
 103 #define LONG_LT_R               "ltr"
 104 #define LONG_CMP                "c"
 105 #define LONG_CMP_R              "cr"
 106 #define LONG_ADDI               "ahi"
 107 #define LONG_ADD_R              "ar"
 108
 109 #endif
 110
 111 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
 112         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
 113                                 (post_commit_ip - start_ip), abort_ip)
 114
 115 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
 116                 RSEQ_INJECT_ASM(1)                                      \
 117                 "larl %%r0, " __rseq_str(cs_label) "\n\t"               \
 118                 LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t"          \
 119                 __rseq_str(label) ":\n\t"
 120
 121 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)              \
 122                 RSEQ_INJECT_ASM(2)                                      \
 123                 "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
 124                 "jnz " __rseq_str(label) "\n\t"
 125
 126 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
 127                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 128                 ".long " __rseq_str(RSEQ_SIG) "\n\t"                    \
 129                 __rseq_str(label) ":\n\t"                               \
 130                 teardown                                                \
 131                 "jg %l[" __rseq_str(abort_label) "]\n\t"                \
 132                 ".popsection\n\t"
 133
 134 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)         \
 135                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 136                 __rseq_str(label) ":\n\t"                               \
 137                 teardown                                                \
 138                 "jg %l[" __rseq_str(cmpfail_label) "]\n\t"              \
 139                 ".popsection\n\t"
 140
 141 static inline __attribute__((always_inline))
 142 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 143 {
 144         RSEQ_INJECT_C(9)
 145
 146         __asm__ __volatile__ goto (
 147                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 148                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 149 #ifdef RSEQ_COMPARE_TWICE
 150                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 151                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 152 #endif
 153                 /* Start rseq by storing table entry pointer into rseq_cs. */
 154                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 155                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 156                 RSEQ_INJECT_ASM(3)
 157                 LONG_CMP " %[expect], %[v]\n\t"
 158                 "jnz %l[cmpfail]\n\t"
 159                 RSEQ_INJECT_ASM(4)
 160 #ifdef RSEQ_COMPARE_TWICE
 161                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 162                 LONG_CMP " %[expect], %[v]\n\t"
 163                 "jnz %l[error2]\n\t"
 164 #endif
 165                 /* final store */
 166                 LONG_S " %[newv], %[v]\n\t"
 167                 "2:\n\t"
 168                 RSEQ_INJECT_ASM(5)
 169                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 170                 : /* gcc asm goto does not allow outputs */
 171                 : [cpu_id]              "r" (cpu),
 172                   [current_cpu_id]      "m" (rseq_get_abi()->cpu_id),
 173                   [rseq_cs]             "m" (rseq_get_abi()->RSEQ_CS_PTR),
 174                   [v]                   "m" (*v),
 175                   [expect]              "r" (expect),
 176                   [newv]                "r" (newv)
 177                   RSEQ_INJECT_INPUT
 178                 : "memory", "cc", "r0"
 179                   RSEQ_INJECT_CLOBBER
 180                 : abort, cmpfail
 181 #ifdef RSEQ_COMPARE_TWICE
 182                   , error1, error2
 183 #endif
 184         );
 185         rseq_after_asm_goto();
 186         return 0;
 187 abort:
 188         rseq_after_asm_goto();
 189         RSEQ_INJECT_FAILED
 190         return -1;
 191 cmpfail:
 192         rseq_after_asm_goto();
 193         return 1;
 194 #ifdef RSEQ_COMPARE_TWICE
 195 error1:
 196         rseq_after_asm_goto();
 197         rseq_bug("cpu_id comparison failed");
 198 error2:
 199         rseq_after_asm_goto();
 200         rseq_bug("expected value comparison failed");
 201 #endif
 202 }
 203
 204 /*
 205  * Compare @v against @expectnot. When it does _not_ match, load @v
 206  * into @load, and store the content of *@v + voffp into @v.
 207  */
 208 static inline __attribute__((always_inline))
 209 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 210                                long voffp, intptr_t *load, int cpu)
 211 {
 212         RSEQ_INJECT_C(9)
 213
 214         __asm__ __volatile__ goto (
 215                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 216                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 217 #ifdef RSEQ_COMPARE_TWICE
 218                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 219                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 220 #endif
 221                 /* Start rseq by storing table entry pointer into rseq_cs. */
 222                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 223                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 224                 RSEQ_INJECT_ASM(3)
 225                 LONG_L " %%r1, %[v]\n\t"
 226                 LONG_CMP_R " %%r1, %[expectnot]\n\t"
 227                 "je %l[cmpfail]\n\t"
 228                 RSEQ_INJECT_ASM(4)
 229 #ifdef RSEQ_COMPARE_TWICE
 230                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 231                 LONG_L " %%r1, %[v]\n\t"
 232                 LONG_CMP_R " %%r1, %[expectnot]\n\t"
 233                 "je %l[error2]\n\t"
 234 #endif
 235                 LONG_S " %%r1, %[load]\n\t"
 236                 LONG_ADD_R " %%r1, %[voffp]\n\t"
 237                 LONG_L " %%r1, 0(%%r1)\n\t"
 238                 /* final store */
 239                 LONG_S " %%r1, %[v]\n\t"
 240                 "2:\n\t"
 241                 RSEQ_INJECT_ASM(5)
 242                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 243                 : /* gcc asm goto does not allow outputs */
 244                 : [cpu_id]              "r" (cpu),
 245                   [current_cpu_id]      "m" (rseq_get_abi()->cpu_id),
 246                   [rseq_cs]             "m" (rseq_get_abi()->RSEQ_CS_PTR),
 247                   /* final store input */
 248                   [v]                   "m" (*v),
 249                   [expectnot]           "r" (expectnot),
 250                   [voffp]               "r" (voffp),
 251                   [load]                "m" (*load)
 252                   RSEQ_INJECT_INPUT
 253                 : "memory", "cc", "r0", "r1"
 254                   RSEQ_INJECT_CLOBBER
 255                 : abort, cmpfail
 256 #ifdef RSEQ_COMPARE_TWICE
 257                   , error1, error2
 258 #endif
 259         );
 260         rseq_after_asm_goto();
 261         return 0;
 262 abort:
 263         rseq_after_asm_goto();
 264         RSEQ_INJECT_FAILED
 265         return -1;
 266 cmpfail:
 267         rseq_after_asm_goto();
 268         return 1;
 269 #ifdef RSEQ_COMPARE_TWICE
 270 error1:
 271         rseq_after_asm_goto();
 272         rseq_bug("cpu_id comparison failed");
 273 error2:
 274         rseq_after_asm_goto();
 275         rseq_bug("expected value comparison failed");
 276 #endif
 277 }
 278
 279 static inline __attribute__((always_inline))
 280 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 281 {
 282         RSEQ_INJECT_C(9)
 283
 284         __asm__ __volatile__ goto (
 285                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 286 #ifdef RSEQ_COMPARE_TWICE
 287                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 288 #endif
 289                 /* Start rseq by storing table entry pointer into rseq_cs. */
 290                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 291                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 292                 RSEQ_INJECT_ASM(3)
 293 #ifdef RSEQ_COMPARE_TWICE
 294                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 295 #endif
 296                 LONG_L " %%r0, %[v]\n\t"
 297                 LONG_ADD_R " %%r0, %[count]\n\t"
 298                 /* final store */
 299                 LONG_S " %%r0, %[v]\n\t"
 300                 "2:\n\t"
 301                 RSEQ_INJECT_ASM(4)
 302                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 303                 : /* gcc asm goto does not allow outputs */
 304                 : [cpu_id]              "r" (cpu),
 305                   [current_cpu_id]      "m" (rseq_get_abi()->cpu_id),
 306                   [rseq_cs]             "m" (rseq_get_abi()->RSEQ_CS_PTR),
 307                   /* final store input */
 308                   [v]                   "m" (*v),
 309                   [count]               "r" (count)
 310                   RSEQ_INJECT_INPUT
 311                 : "memory", "cc", "r0"
 312                   RSEQ_INJECT_CLOBBER
 313                 : abort
 314 #ifdef RSEQ_COMPARE_TWICE
 315                   , error1
 316 #endif
 317         );
 318         rseq_after_asm_goto();
 319         return 0;
 320 abort:
 321         rseq_after_asm_goto();
 322         RSEQ_INJECT_FAILED
 323         return -1;
 324 #ifdef RSEQ_COMPARE_TWICE
 325 error1:
 326         rseq_after_asm_goto();
 327         rseq_bug("cpu_id comparison failed");
 328 #endif
 329 }
 330
 331 static inline __attribute__((always_inline))
 332 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 333                                  intptr_t *v2, intptr_t newv2,
 334                                  intptr_t newv, int cpu)
 335 {
 336         RSEQ_INJECT_C(9)
 337
 338         __asm__ __volatile__ goto (
 339                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 340                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 341 #ifdef RSEQ_COMPARE_TWICE
 342                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 343                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 344 #endif
 345                 /* Start rseq by storing table entry pointer into rseq_cs. */
 346                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 347                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 348                 RSEQ_INJECT_ASM(3)
 349                 LONG_CMP " %[expect], %[v]\n\t"
 350                 "jnz %l[cmpfail]\n\t"
 351                 RSEQ_INJECT_ASM(4)
 352 #ifdef RSEQ_COMPARE_TWICE
 353                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 354                 LONG_CMP " %[expect], %[v]\n\t"
 355                 "jnz %l[error2]\n\t"
 356 #endif
 357                 /* try store */
 358                 LONG_S " %[newv2], %[v2]\n\t"
 359                 RSEQ_INJECT_ASM(5)
 360                 /* final store */
 361                 LONG_S " %[newv], %[v]\n\t"
 362                 "2:\n\t"
 363                 RSEQ_INJECT_ASM(6)
 364                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 365                 : /* gcc asm goto does not allow outputs */
 366                 : [cpu_id]              "r" (cpu),
 367                   [current_cpu_id]      "m" (rseq_get_abi()->cpu_id),
 368                   [rseq_cs]             "m" (rseq_get_abi()->RSEQ_CS_PTR),
 369                   /* try store input */
 370                   [v2]                  "m" (*v2),
 371                   [newv2]               "r" (newv2),
 372                   /* final store input */
 373                   [v]                   "m" (*v),
 374                   [expect]              "r" (expect),
 375                   [newv]                "r" (newv)
 376                   RSEQ_INJECT_INPUT
 377                 : "memory", "cc", "r0"
 378                   RSEQ_INJECT_CLOBBER
 379                 : abort, cmpfail
 380 #ifdef RSEQ_COMPARE_TWICE
 381                   , error1, error2
 382 #endif
 383         );
 384         rseq_after_asm_goto();
 385         return 0;
 386 abort:
 387         rseq_after_asm_goto();
 388         RSEQ_INJECT_FAILED
 389         return -1;
 390 cmpfail:
 391         rseq_after_asm_goto();
 392         return 1;
 393 #ifdef RSEQ_COMPARE_TWICE
 394 error1:
 395         rseq_after_asm_goto();
 396         rseq_bug("cpu_id comparison failed");
 397 error2:
 398         rseq_after_asm_goto();
 399         rseq_bug("expected value comparison failed");
 400 #endif
 401 }
 402
 403 /* s390 is TSO. */
 404 static inline __attribute__((always_inline))
 405 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 406                                          intptr_t *v2, intptr_t newv2,
 407                                          intptr_t newv, int cpu)
 408 {
 409         return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
 410 }
 411
 412 static inline __attribute__((always_inline))
 413 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 414                               intptr_t *v2, intptr_t expect2,
 415                               intptr_t newv, int cpu)
 416 {
 417         RSEQ_INJECT_C(9)
 418
 419         __asm__ __volatile__ goto (
 420                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 421                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 422 #ifdef RSEQ_COMPARE_TWICE
 423                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 424                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 425                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
 426 #endif
 427                 /* Start rseq by storing table entry pointer into rseq_cs. */
 428                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 429                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 430                 RSEQ_INJECT_ASM(3)
 431                 LONG_CMP " %[expect], %[v]\n\t"
 432                 "jnz %l[cmpfail]\n\t"
 433                 RSEQ_INJECT_ASM(4)
 434                 LONG_CMP " %[expect2], %[v2]\n\t"
 435                 "jnz %l[cmpfail]\n\t"
 436                 RSEQ_INJECT_ASM(5)
 437 #ifdef RSEQ_COMPARE_TWICE
 438                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 439                 LONG_CMP " %[expect], %[v]\n\t"
 440                 "jnz %l[error2]\n\t"
 441                 LONG_CMP " %[expect2], %[v2]\n\t"
 442                 "jnz %l[error3]\n\t"
 443 #endif
 444                 /* final store */
 445                 LONG_S " %[newv], %[v]\n\t"
 446                 "2:\n\t"
 447                 RSEQ_INJECT_ASM(6)
 448                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 449                 : /* gcc asm goto does not allow outputs */
 450                 : [cpu_id]              "r" (cpu),
 451                   [current_cpu_id]      "m" (rseq_get_abi()->cpu_id),
 452                   [rseq_cs]             "m" (rseq_get_abi()->RSEQ_CS_PTR),
 453                   /* cmp2 input */
 454                   [v2]                  "m" (*v2),
 455                   [expect2]             "r" (expect2),
 456                   /* final store input */
 457                   [v]                   "m" (*v),
 458                   [expect]              "r" (expect),
 459                   [newv]                "r" (newv)
 460                   RSEQ_INJECT_INPUT
 461                 : "memory", "cc", "r0"
 462                   RSEQ_INJECT_CLOBBER
 463                 : abort, cmpfail
 464 #ifdef RSEQ_COMPARE_TWICE
 465                   , error1, error2, error3
 466 #endif
 467         );
 468         rseq_after_asm_goto();
 469         return 0;
 470 abort:
 471         rseq_after_asm_goto();
 472         RSEQ_INJECT_FAILED
 473         return -1;
 474 cmpfail:
 475         rseq_after_asm_goto();
 476         return 1;
 477 #ifdef RSEQ_COMPARE_TWICE
 478 error1:
 479         rseq_after_asm_goto();
 480         rseq_bug("cpu_id comparison failed");
 481 error2:
 482         rseq_after_asm_goto();
 483         rseq_bug("1st expected value comparison failed");
 484 error3:
 485         rseq_after_asm_goto();
 486         rseq_bug("2nd expected value comparison failed");
 487 #endif
 488 }
 489
 490 static inline __attribute__((always_inline))
 491 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 492                                  void *dst, void *src, size_t len,
 493                                  intptr_t newv, int cpu)
 494 {
 495         uint64_t rseq_scratch[3];
 496
 497         RSEQ_INJECT_C(9)
 498
 499         __asm__ __volatile__ goto (
 500                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 501                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 502 #ifdef RSEQ_COMPARE_TWICE
 503                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 504                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 505 #endif
 506                 LONG_S " %[src], %[rseq_scratch0]\n\t"
 507                 LONG_S " %[dst], %[rseq_scratch1]\n\t"
 508                 LONG_S " %[len], %[rseq_scratch2]\n\t"
 509                 /* Start rseq by storing table entry pointer into rseq_cs. */
 510                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 511                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 512                 RSEQ_INJECT_ASM(3)
 513                 LONG_CMP " %[expect], %[v]\n\t"
 514                 "jnz 5f\n\t"
 515                 RSEQ_INJECT_ASM(4)
 516 #ifdef RSEQ_COMPARE_TWICE
 517                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
 518                 LONG_CMP " %[expect], %[v]\n\t"
 519                 "jnz 7f\n\t"
 520 #endif
 521                 /* try memcpy */
 522                 LONG_LT_R " %[len], %[len]\n\t"
 523                 "jz 333f\n\t"
 524                 "222:\n\t"
 525                 "ic %%r0,0(%[src])\n\t"
 526                 "stc %%r0,0(%[dst])\n\t"
 527                 LONG_ADDI " %[src], 1\n\t"
 528                 LONG_ADDI " %[dst], 1\n\t"
 529                 LONG_ADDI " %[len], -1\n\t"
 530                 "jnz 222b\n\t"
 531                 "333:\n\t"
 532                 RSEQ_INJECT_ASM(5)
 533                 /* final store */
 534                 LONG_S " %[newv], %[v]\n\t"
 535                 "2:\n\t"
 536                 RSEQ_INJECT_ASM(6)
 537                 /* teardown */
 538                 LONG_L " %[len], %[rseq_scratch2]\n\t"
 539                 LONG_L " %[dst], %[rseq_scratch1]\n\t"
 540                 LONG_L " %[src], %[rseq_scratch0]\n\t"
 541                 RSEQ_ASM_DEFINE_ABORT(4,
 542                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 543                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 544                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 545                         abort)
 546                 RSEQ_ASM_DEFINE_CMPFAIL(5,
 547                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 548                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 549                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 550                         cmpfail)
 551 #ifdef RSEQ_COMPARE_TWICE
 552                 RSEQ_ASM_DEFINE_CMPFAIL(6,
 553                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 554                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 555                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 556                         error1)
 557                 RSEQ_ASM_DEFINE_CMPFAIL(7,
 558                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 559                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 560                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 561                         error2)
 562 #endif
 563                 : /* gcc asm goto does not allow outputs */
 564                 : [cpu_id]              "r" (cpu),
 565                   [current_cpu_id]      "m" (rseq_get_abi()->cpu_id),
 566                   [rseq_cs]             "m" (rseq_get_abi()->RSEQ_CS_PTR),
 567                   /* final store input */
 568                   [v]                   "m" (*v),
 569                   [expect]              "r" (expect),
 570                   [newv]                "r" (newv),
 571                   /* try memcpy input */
 572                   [dst]                 "r" (dst),
 573                   [src]                 "r" (src),
 574                   [len]                 "r" (len),
 575                   [rseq_scratch0]       "m" (rseq_scratch[0]),
 576                   [rseq_scratch1]       "m" (rseq_scratch[1]),
 577                   [rseq_scratch2]       "m" (rseq_scratch[2])
 578                   RSEQ_INJECT_INPUT
 579                 : "memory", "cc", "r0"
 580                   RSEQ_INJECT_CLOBBER
 581                 : abort, cmpfail
 582 #ifdef RSEQ_COMPARE_TWICE
 583                   , error1, error2
 584 #endif
 585         );
 586         rseq_after_asm_goto();
 587         return 0;
 588 abort:
 589         rseq_after_asm_goto();
 590         RSEQ_INJECT_FAILED
 591         return -1;
 592 cmpfail:
 593         rseq_after_asm_goto();
 594         return 1;
 595 #ifdef RSEQ_COMPARE_TWICE
 596 error1:
 597         rseq_after_asm_goto();
 598         rseq_bug("cpu_id comparison failed");
 599 error2:
 600         rseq_after_asm_goto();
 601         rseq_bug("expected value comparison failed");
 602 #endif
 603 }
 604
 605 /* s390 is TSO. */
 606 static inline __attribute__((always_inline))
 607 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
 608                                          void *dst, void *src, size_t len,
 609                                          intptr_t newv, int cpu)
 610 {
 611         return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
 612                                             newv, cpu);
 613 }
 614
 615 #endif /* !RSEQ_SKIP_FASTPATH */