include/rseq/rseq-s390.h

   1 /* SPDX-License-Identifier: MIT */
   2 /* SPDX-FileCopyrightText: 2018 Vasily Gorbik <gor@linux.ibm.com> */
   3
   4 /*
   5  * RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the
   6  * access-register mode nor the linkage stack this instruction will always
   7  * cause a special-operation exception (the trap-enabled bit in the DUCT
   8  * is and will stay 0). The instruction pattern is
   9  *      b2 ff 0f ff     trap4   4095(%r0)
  10  */
  11 #define RSEQ_SIG        0xB2FF0FFF
  12
  13 #define rseq_smp_mb()   __asm__ __volatile__ ("bcr 15,0" ::: "memory")
  14 #define rseq_smp_rmb()  rseq_smp_mb()
  15 #define rseq_smp_wmb()  rseq_smp_mb()
  16
  17 #define rseq_smp_load_acquire(p)                                        \
  18 __extension__ ({                                                        \
  19         __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p);                       \
  20         rseq_barrier();                                                 \
  21         ____p1;                                                         \
  22 })
  23
  24 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  25
  26 #define rseq_smp_store_release(p, v)                                    \
  27 do {                                                                    \
  28         rseq_barrier();                                                 \
  29         RSEQ_WRITE_ONCE(*p, v);                                         \
  30 } while (0)
  31
  32 #ifdef RSEQ_SKIP_FASTPATH
  33 #include "rseq-skip.h"
  34 #else /* !RSEQ_SKIP_FASTPATH */
  35
  36 #ifdef __s390x__
  37
  38 #define LONG_L                  "lg"
  39 #define LONG_S                  "stg"
  40 #define LONG_LT_R               "ltgr"
  41 #define LONG_CMP                "cg"
  42 #define LONG_CMP_R              "cgr"
  43 #define LONG_ADDI               "aghi"
  44 #define LONG_ADD_R              "agr"
  45
  46 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
  47                                 start_ip, post_commit_offset, abort_ip) \
  48                 ".pushsection __rseq_cs, \"aw\"\n\t"                    \
  49                 ".balign 32\n\t"                                        \
  50                 __rseq_str(label) ":\n\t"                               \
  51                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  52                 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
  53                 ".popsection\n\t"                                       \
  54                 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
  55                 ".quad " __rseq_str(label) "b\n\t"                      \
  56                 ".popsection\n\t"
  57
  58 /*
  59  * Exit points of a rseq critical section consist of all instructions outside
  60  * of the critical section where a critical section can either branch to or
  61  * reach through the normal course of its execution. The abort IP and the
  62  * post-commit IP are already part of the __rseq_cs section and should not be
  63  * explicitly defined as additional exit points. Knowing all exit points is
  64  * useful to assist debuggers stepping over the critical section.
  65  */
  66 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
  67                 ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
  68                 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
  69                 ".popsection\n\t"
  70
  71 #elif __s390__
  72
  73 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
  74                                 start_ip, post_commit_offset, abort_ip) \
  75                 ".pushsection __rseq_cs, \"aw\"\n\t"                    \
  76                 ".balign 32\n\t"                                        \
  77                 __rseq_str(label) ":\n\t"                               \
  78                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  79                 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
  80                 ".popsection\n\t"                                       \
  81                 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
  82                 ".long 0x0, " __rseq_str(label) "b\n\t"                 \
  83                 ".popsection\n\t"
  84
  85 /*
  86  * Exit points of a rseq critical section consist of all instructions outside
  87  * of the critical section where a critical section can either branch to or
  88  * reach through the normal course of its execution. The abort IP and the
  89  * post-commit IP are already part of the __rseq_cs section and should not be
  90  * explicitly defined as additional exit points. Knowing all exit points is
  91  * useful to assist debuggers stepping over the critical section.
  92  */
  93 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
  94                 ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
  95                 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
  96                 ".popsection\n\t"
  97
  98 #define LONG_L                  "l"
  99 #define LONG_S                  "st"
 100 #define LONG_LT_R               "ltr"
 101 #define LONG_CMP                "c"
 102 #define LONG_CMP_R              "cr"
 103 #define LONG_ADDI               "ahi"
 104 #define LONG_ADD_R              "ar"
 105
 106 #endif
 107
 108 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
 109         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
 110                                 (post_commit_ip - start_ip), abort_ip)
 111
 112 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
 113                 RSEQ_INJECT_ASM(1)                                      \
 114                 "larl %%r0, " __rseq_str(cs_label) "\n\t"               \
 115                 LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t"          \
 116                 __rseq_str(label) ":\n\t"
 117
 118 #define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label)              \
 119                 RSEQ_INJECT_ASM(2)                                      \
 120                 "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
 121                 "jnz " __rseq_str(label) "\n\t"
 122
 123 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
 124                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 125                 ".long " __rseq_str(RSEQ_SIG) "\n\t"                    \
 126                 __rseq_str(label) ":\n\t"                               \
 127                 teardown                                                \
 128                 "jg %l[" __rseq_str(abort_label) "]\n\t"                \
 129                 ".popsection\n\t"
 130
 131 #define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label)         \
 132                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 133                 __rseq_str(label) ":\n\t"                               \
 134                 teardown                                                \
 135                 "jg %l[" __rseq_str(cmpfail_label) "]\n\t"              \
 136                 ".popsection\n\t"
 137
 138 static inline __attribute__((always_inline))
 139 int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
 140 {
 141         RSEQ_INJECT_C(9)
 142
 143         __asm__ __volatile__ goto (
 144                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 145                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 146 #ifdef RSEQ_COMPARE_TWICE
 147                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 148                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 149 #endif
 150                 /* Start rseq by storing table entry pointer into rseq_cs. */
 151                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 152                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 153                 RSEQ_INJECT_ASM(3)
 154                 LONG_CMP " %[expect], %[v]\n\t"
 155                 "jnz %l[cmpfail]\n\t"
 156                 RSEQ_INJECT_ASM(4)
 157 #ifdef RSEQ_COMPARE_TWICE
 158                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 159                 LONG_CMP " %[expect], %[v]\n\t"
 160                 "jnz %l[error2]\n\t"
 161 #endif
 162                 /* final store */
 163                 LONG_S " %[newv], %[v]\n\t"
 164                 "2:\n\t"
 165                 RSEQ_INJECT_ASM(5)
 166                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 167                 : /* gcc asm goto does not allow outputs */
 168                 : [cpu_id]              "r" (cpu),
 169                   [current_cpu_id]      "m" (rseq_get_abi()->cpu_id),
 170                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 171                   [v]                   "m" (*v),
 172                   [expect]              "r" (expect),
 173                   [newv]                "r" (newv)
 174                   RSEQ_INJECT_INPUT
 175                 : "memory", "cc", "r0"
 176                   RSEQ_INJECT_CLOBBER
 177                 : abort, cmpfail
 178 #ifdef RSEQ_COMPARE_TWICE
 179                   , error1, error2
 180 #endif
 181         );
 182         rseq_after_asm_goto();
 183         return 0;
 184 abort:
 185         rseq_after_asm_goto();
 186         RSEQ_INJECT_FAILED
 187         return -1;
 188 cmpfail:
 189         rseq_after_asm_goto();
 190         return 1;
 191 #ifdef RSEQ_COMPARE_TWICE
 192 error1:
 193         rseq_after_asm_goto();
 194         rseq_bug("cpu_id comparison failed");
 195 error2:
 196         rseq_after_asm_goto();
 197         rseq_bug("expected value comparison failed");
 198 #endif
 199 }
 200
 201 /*
 202  * Compare @v against @expectnot. When it does _not_ match, load @v
 203  * into @load, and store the content of *@v + voffp into @v.
 204  */
 205 static inline __attribute__((always_inline))
 206 int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
 207                                long voffp, intptr_t *load, int cpu)
 208 {
 209         RSEQ_INJECT_C(9)
 210
 211         __asm__ __volatile__ goto (
 212                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 213                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 214 #ifdef RSEQ_COMPARE_TWICE
 215                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 216                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 217 #endif
 218                 /* Start rseq by storing table entry pointer into rseq_cs. */
 219                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 220                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 221                 RSEQ_INJECT_ASM(3)
 222                 LONG_L " %%r1, %[v]\n\t"
 223                 LONG_CMP_R " %%r1, %[expectnot]\n\t"
 224                 "je %l[cmpfail]\n\t"
 225                 RSEQ_INJECT_ASM(4)
 226 #ifdef RSEQ_COMPARE_TWICE
 227                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 228                 LONG_L " %%r1, %[v]\n\t"
 229                 LONG_CMP_R " %%r1, %[expectnot]\n\t"
 230                 "je %l[error2]\n\t"
 231 #endif
 232                 LONG_S " %%r1, %[load]\n\t"
 233                 LONG_ADD_R " %%r1, %[voffp]\n\t"
 234                 LONG_L " %%r1, 0(%%r1)\n\t"
 235                 /* final store */
 236                 LONG_S " %%r1, %[v]\n\t"
 237                 "2:\n\t"
 238                 RSEQ_INJECT_ASM(5)
 239                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 240                 : /* gcc asm goto does not allow outputs */
 241                 : [cpu_id]              "r" (cpu),
 242                   [current_cpu_id]      "m" (rseq_get_abi()->cpu_id),
 243                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 244                   /* final store input */
 245                   [v]                   "m" (*v),
 246                   [expectnot]           "r" (expectnot),
 247                   [voffp]               "r" (voffp),
 248                   [load]                "m" (*load)
 249                   RSEQ_INJECT_INPUT
 250                 : "memory", "cc", "r0", "r1"
 251                   RSEQ_INJECT_CLOBBER
 252                 : abort, cmpfail
 253 #ifdef RSEQ_COMPARE_TWICE
 254                   , error1, error2
 255 #endif
 256         );
 257         rseq_after_asm_goto();
 258         return 0;
 259 abort:
 260         rseq_after_asm_goto();
 261         RSEQ_INJECT_FAILED
 262         return -1;
 263 cmpfail:
 264         rseq_after_asm_goto();
 265         return 1;
 266 #ifdef RSEQ_COMPARE_TWICE
 267 error1:
 268         rseq_after_asm_goto();
 269         rseq_bug("cpu_id comparison failed");
 270 error2:
 271         rseq_after_asm_goto();
 272         rseq_bug("expected value comparison failed");
 273 #endif
 274 }
 275
 276 static inline __attribute__((always_inline))
 277 int rseq_addv(intptr_t *v, intptr_t count, int cpu)
 278 {
 279         RSEQ_INJECT_C(9)
 280
 281         __asm__ __volatile__ goto (
 282                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 283 #ifdef RSEQ_COMPARE_TWICE
 284                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 285 #endif
 286                 /* Start rseq by storing table entry pointer into rseq_cs. */
 287                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 288                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 289                 RSEQ_INJECT_ASM(3)
 290 #ifdef RSEQ_COMPARE_TWICE
 291                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 292 #endif
 293                 LONG_L " %%r0, %[v]\n\t"
 294                 LONG_ADD_R " %%r0, %[count]\n\t"
 295                 /* final store */
 296                 LONG_S " %%r0, %[v]\n\t"
 297                 "2:\n\t"
 298                 RSEQ_INJECT_ASM(4)
 299                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 300                 : /* gcc asm goto does not allow outputs */
 301                 : [cpu_id]              "r" (cpu),
 302                   [current_cpu_id]      "m" (rseq_get_abi()->cpu_id),
 303                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 304                   /* final store input */
 305                   [v]                   "m" (*v),
 306                   [count]               "r" (count)
 307                   RSEQ_INJECT_INPUT
 308                 : "memory", "cc", "r0"
 309                   RSEQ_INJECT_CLOBBER
 310                 : abort
 311 #ifdef RSEQ_COMPARE_TWICE
 312                   , error1
 313 #endif
 314         );
 315         rseq_after_asm_goto();
 316         return 0;
 317 abort:
 318         rseq_after_asm_goto();
 319         RSEQ_INJECT_FAILED
 320         return -1;
 321 #ifdef RSEQ_COMPARE_TWICE
 322 error1:
 323         rseq_after_asm_goto();
 324         rseq_bug("cpu_id comparison failed");
 325 #endif
 326 }
 327
 328 static inline __attribute__((always_inline))
 329 int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
 330                                  intptr_t *v2, intptr_t newv2,
 331                                  intptr_t newv, int cpu)
 332 {
 333         RSEQ_INJECT_C(9)
 334
 335         __asm__ __volatile__ goto (
 336                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 337                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 338 #ifdef RSEQ_COMPARE_TWICE
 339                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 340                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 341 #endif
 342                 /* Start rseq by storing table entry pointer into rseq_cs. */
 343                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 344                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 345                 RSEQ_INJECT_ASM(3)
 346                 LONG_CMP " %[expect], %[v]\n\t"
 347                 "jnz %l[cmpfail]\n\t"
 348                 RSEQ_INJECT_ASM(4)
 349 #ifdef RSEQ_COMPARE_TWICE
 350                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 351                 LONG_CMP " %[expect], %[v]\n\t"
 352                 "jnz %l[error2]\n\t"
 353 #endif
 354                 /* try store */
 355                 LONG_S " %[newv2], %[v2]\n\t"
 356                 RSEQ_INJECT_ASM(5)
 357                 /* final store */
 358                 LONG_S " %[newv], %[v]\n\t"
 359                 "2:\n\t"
 360                 RSEQ_INJECT_ASM(6)
 361                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 362                 : /* gcc asm goto does not allow outputs */
 363                 : [cpu_id]              "r" (cpu),
 364                   [current_cpu_id]      "m" (rseq_get_abi()->cpu_id),
 365                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 366                   /* try store input */
 367                   [v2]                  "m" (*v2),
 368                   [newv2]               "r" (newv2),
 369                   /* final store input */
 370                   [v]                   "m" (*v),
 371                   [expect]              "r" (expect),
 372                   [newv]                "r" (newv)
 373                   RSEQ_INJECT_INPUT
 374                 : "memory", "cc", "r0"
 375                   RSEQ_INJECT_CLOBBER
 376                 : abort, cmpfail
 377 #ifdef RSEQ_COMPARE_TWICE
 378                   , error1, error2
 379 #endif
 380         );
 381         rseq_after_asm_goto();
 382         return 0;
 383 abort:
 384         rseq_after_asm_goto();
 385         RSEQ_INJECT_FAILED
 386         return -1;
 387 cmpfail:
 388         rseq_after_asm_goto();
 389         return 1;
 390 #ifdef RSEQ_COMPARE_TWICE
 391 error1:
 392         rseq_after_asm_goto();
 393         rseq_bug("cpu_id comparison failed");
 394 error2:
 395         rseq_after_asm_goto();
 396         rseq_bug("expected value comparison failed");
 397 #endif
 398 }
 399
 400 /* s390 is TSO. */
 401 static inline __attribute__((always_inline))
 402 int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
 403                                          intptr_t *v2, intptr_t newv2,
 404                                          intptr_t newv, int cpu)
 405 {
 406         return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
 407 }
 408
 409 static inline __attribute__((always_inline))
 410 int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
 411                               intptr_t *v2, intptr_t expect2,
 412                               intptr_t newv, int cpu)
 413 {
 414         RSEQ_INJECT_C(9)
 415
 416         __asm__ __volatile__ goto (
 417                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 418                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 419 #ifdef RSEQ_COMPARE_TWICE
 420                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 421                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 422                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
 423 #endif
 424                 /* Start rseq by storing table entry pointer into rseq_cs. */
 425                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 426                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 427                 RSEQ_INJECT_ASM(3)
 428                 LONG_CMP " %[expect], %[v]\n\t"
 429                 "jnz %l[cmpfail]\n\t"
 430                 RSEQ_INJECT_ASM(4)
 431                 LONG_CMP " %[expect2], %[v2]\n\t"
 432                 "jnz %l[cmpfail]\n\t"
 433                 RSEQ_INJECT_ASM(5)
 434 #ifdef RSEQ_COMPARE_TWICE
 435                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 436                 LONG_CMP " %[expect], %[v]\n\t"
 437                 "jnz %l[error2]\n\t"
 438                 LONG_CMP " %[expect2], %[v2]\n\t"
 439                 "jnz %l[error3]\n\t"
 440 #endif
 441                 /* final store */
 442                 LONG_S " %[newv], %[v]\n\t"
 443                 "2:\n\t"
 444                 RSEQ_INJECT_ASM(6)
 445                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 446                 : /* gcc asm goto does not allow outputs */
 447                 : [cpu_id]              "r" (cpu),
 448                   [current_cpu_id]      "m" (rseq_get_abi()->cpu_id),
 449                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 450                   /* cmp2 input */
 451                   [v2]                  "m" (*v2),
 452                   [expect2]             "r" (expect2),
 453                   /* final store input */
 454                   [v]                   "m" (*v),
 455                   [expect]              "r" (expect),
 456                   [newv]                "r" (newv)
 457                   RSEQ_INJECT_INPUT
 458                 : "memory", "cc", "r0"
 459                   RSEQ_INJECT_CLOBBER
 460                 : abort, cmpfail
 461 #ifdef RSEQ_COMPARE_TWICE
 462                   , error1, error2, error3
 463 #endif
 464         );
 465         rseq_after_asm_goto();
 466         return 0;
 467 abort:
 468         rseq_after_asm_goto();
 469         RSEQ_INJECT_FAILED
 470         return -1;
 471 cmpfail:
 472         rseq_after_asm_goto();
 473         return 1;
 474 #ifdef RSEQ_COMPARE_TWICE
 475 error1:
 476         rseq_after_asm_goto();
 477         rseq_bug("cpu_id comparison failed");
 478 error2:
 479         rseq_after_asm_goto();
 480         rseq_bug("1st expected value comparison failed");
 481 error3:
 482         rseq_after_asm_goto();
 483         rseq_bug("2nd expected value comparison failed");
 484 #endif
 485 }
 486
 487 static inline __attribute__((always_inline))
 488 int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
 489                                  void *dst, void *src, size_t len,
 490                                  intptr_t newv, int cpu)
 491 {
 492         uint64_t rseq_scratch[3];
 493
 494         RSEQ_INJECT_C(9)
 495
 496         __asm__ __volatile__ goto (
 497                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 498                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 499 #ifdef RSEQ_COMPARE_TWICE
 500                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 501                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 502 #endif
 503                 LONG_S " %[src], %[rseq_scratch0]\n\t"
 504                 LONG_S " %[dst], %[rseq_scratch1]\n\t"
 505                 LONG_S " %[len], %[rseq_scratch2]\n\t"
 506                 /* Start rseq by storing table entry pointer into rseq_cs. */
 507                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 508                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
 509                 RSEQ_INJECT_ASM(3)
 510                 LONG_CMP " %[expect], %[v]\n\t"
 511                 "jnz 5f\n\t"
 512                 RSEQ_INJECT_ASM(4)
 513 #ifdef RSEQ_COMPARE_TWICE
 514                 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
 515                 LONG_CMP " %[expect], %[v]\n\t"
 516                 "jnz 7f\n\t"
 517 #endif
 518                 /* try memcpy */
 519                 LONG_LT_R " %[len], %[len]\n\t"
 520                 "jz 333f\n\t"
 521                 "222:\n\t"
 522                 "ic %%r0,0(%[src])\n\t"
 523                 "stc %%r0,0(%[dst])\n\t"
 524                 LONG_ADDI " %[src], 1\n\t"
 525                 LONG_ADDI " %[dst], 1\n\t"
 526                 LONG_ADDI " %[len], -1\n\t"
 527                 "jnz 222b\n\t"
 528                 "333:\n\t"
 529                 RSEQ_INJECT_ASM(5)
 530                 /* final store */
 531                 LONG_S " %[newv], %[v]\n\t"
 532                 "2:\n\t"
 533                 RSEQ_INJECT_ASM(6)
 534                 /* teardown */
 535                 LONG_L " %[len], %[rseq_scratch2]\n\t"
 536                 LONG_L " %[dst], %[rseq_scratch1]\n\t"
 537                 LONG_L " %[src], %[rseq_scratch0]\n\t"
 538                 RSEQ_ASM_DEFINE_ABORT(4,
 539                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 540                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 541                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 542                         abort)
 543                 RSEQ_ASM_DEFINE_CMPFAIL(5,
 544                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 545                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 546                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 547                         cmpfail)
 548 #ifdef RSEQ_COMPARE_TWICE
 549                 RSEQ_ASM_DEFINE_CMPFAIL(6,
 550                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 551                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 552                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 553                         error1)
 554                 RSEQ_ASM_DEFINE_CMPFAIL(7,
 555                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 556                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 557                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 558                         error2)
 559 #endif
 560                 : /* gcc asm goto does not allow outputs */
 561                 : [cpu_id]              "r" (cpu),
 562                   [current_cpu_id]      "m" (rseq_get_abi()->cpu_id),
 563                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 564                   /* final store input */
 565                   [v]                   "m" (*v),
 566                   [expect]              "r" (expect),
 567                   [newv]                "r" (newv),
 568                   /* try memcpy input */
 569                   [dst]                 "r" (dst),
 570                   [src]                 "r" (src),
 571                   [len]                 "r" (len),
 572                   [rseq_scratch0]       "m" (rseq_scratch[0]),
 573                   [rseq_scratch1]       "m" (rseq_scratch[1]),
 574                   [rseq_scratch2]       "m" (rseq_scratch[2])
 575                   RSEQ_INJECT_INPUT
 576                 : "memory", "cc", "r0"
 577                   RSEQ_INJECT_CLOBBER
 578                 : abort, cmpfail
 579 #ifdef RSEQ_COMPARE_TWICE
 580                   , error1, error2
 581 #endif
 582         );
 583         rseq_after_asm_goto();
 584         return 0;
 585 abort:
 586         rseq_after_asm_goto();
 587         RSEQ_INJECT_FAILED
 588         return -1;
 589 cmpfail:
 590         rseq_after_asm_goto();
 591         return 1;
 592 #ifdef RSEQ_COMPARE_TWICE
 593 error1:
 594         rseq_after_asm_goto();
 595         rseq_bug("cpu_id comparison failed");
 596 error2:
 597         rseq_after_asm_goto();
 598         rseq_bug("expected value comparison failed");
 599 #endif
 600 }
 601
 602 /* s390 is TSO. */
 603 static inline __attribute__((always_inline))
 604 int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
 605                                          void *dst, void *src, size_t len,
 606                                          intptr_t newv, int cpu)
 607 {
 608         return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
 609                                             newv, cpu);
 610 }
 611
 612 #endif /* !RSEQ_SKIP_FASTPATH */