include/rseq/arch/arm/bits.h

   1 /* SPDX-License-Identifier: MIT */
   2 /* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
   3
   4 /*
   5  * rseq/arch/arm/bits.h
   6  */
   7
   8 #include "rseq/arch/templates/bits.h"
   9
  10 /*
  11  * Refer to rseq/pseudocode.h for documentation and pseudo-code of the
  12  * rseq critical section helpers.
  13  */
  14 #include "rseq/pseudocode.h"
  15
  16 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
  17         (defined(RSEQ_TEMPLATE_INDEX_CPU_ID) || defined(RSEQ_TEMPLATE_INDEX_MM_CID))
  18
  19 static inline __attribute__((always_inline))
  20 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_store__ptr)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
  21 {
  22         RSEQ_INJECT_C(9)
  23
  24         __asm__ __volatile__ goto (
  25                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
  26                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[ne])
  27 #ifdef RSEQ_COMPARE_TWICE
  28                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
  29                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
  30 #endif
  31                 /* Start rseq by storing table entry pointer into rseq_cs. */
  32                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
  33                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
  34                 RSEQ_INJECT_ASM(3)
  35                 "ldr r0, %[v]\n\t"
  36                 "cmp %[expect], r0\n\t"
  37                 "bne %l[ne]\n\t"
  38                 RSEQ_INJECT_ASM(4)
  39 #ifdef RSEQ_COMPARE_TWICE
  40                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
  41                 "ldr r0, %[v]\n\t"
  42                 "cmp %[expect], r0\n\t"
  43                 "bne %l[error2]\n\t"
  44 #endif
  45                 /* final store */
  46                 "str %[newv], %[v]\n\t"
  47                 "2:\n\t"
  48                 RSEQ_INJECT_ASM(5)
  49                 "b 5f\n\t"
  50                 RSEQ_ASM_DEFINE_ABORT(4, "", abort, 3, 1b, 2b, 4f)
  51                 "5:\n\t"
  52                 : /* gcc asm goto does not allow outputs */
  53                 : [cpu_id]              "r" (cpu),
  54                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_INDEX_CPU_ID_FIELD),
  55                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
  56                   [v]                   "m" (*v),
  57                   [expect]              "r" (expect),
  58                   [newv]                "r" (newv)
  59                   RSEQ_INJECT_INPUT
  60                 : "r0", "memory", "cc"
  61                   RSEQ_INJECT_CLOBBER
  62                 : abort, ne
  63 #ifdef RSEQ_COMPARE_TWICE
  64                   , error1, error2
  65 #endif
  66         );
  67         rseq_after_asm_goto();
  68         return 0;
  69 abort:
  70         rseq_after_asm_goto();
  71         RSEQ_INJECT_FAILED
  72         return -1;
  73 ne:
  74         rseq_after_asm_goto();
  75         return 1;
  76 #ifdef RSEQ_COMPARE_TWICE
  77 error1:
  78         rseq_after_asm_goto();
  79         rseq_bug("cpu_id comparison failed");
  80 error2:
  81         rseq_after_asm_goto();
  82         rseq_bug("expected value comparison failed");
  83 #endif
  84 }
  85
  86 static inline __attribute__((always_inline))
  87 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbeq_store_add_load_store__ptr)(intptr_t *v, intptr_t expectnot,
  88                                long voffp, intptr_t *load, int cpu)
  89 {
  90         RSEQ_INJECT_C(9)
  91
  92         __asm__ __volatile__ goto (
  93                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
  94                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[eq])
  95 #ifdef RSEQ_COMPARE_TWICE
  96                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
  97                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
  98 #endif
  99                 /* Start rseq by storing table entry pointer into rseq_cs. */
 100                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 101                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
 102                 RSEQ_INJECT_ASM(3)
 103                 "ldr r0, %[v]\n\t"
 104                 "cmp %[expectnot], r0\n\t"
 105                 "beq %l[eq]\n\t"
 106                 RSEQ_INJECT_ASM(4)
 107 #ifdef RSEQ_COMPARE_TWICE
 108                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 109                 "ldr r0, %[v]\n\t"
 110                 "cmp %[expectnot], r0\n\t"
 111                 "beq %l[error2]\n\t"
 112 #endif
 113                 "str r0, %[load]\n\t"
 114                 "add r0, %[voffp]\n\t"
 115                 "ldr r0, [r0]\n\t"
 116                 /* final store */
 117                 "str r0, %[v]\n\t"
 118                 "2:\n\t"
 119                 RSEQ_INJECT_ASM(5)
 120                 "b 5f\n\t"
 121                 RSEQ_ASM_DEFINE_ABORT(4, "", abort, 3, 1b, 2b, 4f)
 122                 "5:\n\t"
 123                 : /* gcc asm goto does not allow outputs */
 124                 : [cpu_id]              "r" (cpu),
 125                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_INDEX_CPU_ID_FIELD),
 126                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 127                   /* final store input */
 128                   [v]                   "m" (*v),
 129                   [expectnot]           "r" (expectnot),
 130                   [voffp]               "Ir" (voffp),
 131                   [load]                "m" (*load)
 132                   RSEQ_INJECT_INPUT
 133                 : "r0", "memory", "cc"
 134                   RSEQ_INJECT_CLOBBER
 135                 : abort, eq
 136 #ifdef RSEQ_COMPARE_TWICE
 137                   , error1, error2
 138 #endif
 139         );
 140         rseq_after_asm_goto();
 141         return 0;
 142 abort:
 143         rseq_after_asm_goto();
 144         RSEQ_INJECT_FAILED
 145         return -1;
 146 eq:
 147         rseq_after_asm_goto();
 148         return 1;
 149 #ifdef RSEQ_COMPARE_TWICE
 150 error1:
 151         rseq_after_asm_goto();
 152         rseq_bug("cpu_id comparison failed");
 153 error2:
 154         rseq_after_asm_goto();
 155         rseq_bug("expected value comparison failed");
 156 #endif
 157 }
 158
 159 static inline __attribute__((always_inline))
 160 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_add_store__ptr)(intptr_t *v, intptr_t count, int cpu)
 161 {
 162         RSEQ_INJECT_C(9)
 163
 164         __asm__ __volatile__ goto (
 165                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
 166 #ifdef RSEQ_COMPARE_TWICE
 167                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 168 #endif
 169                 /* Start rseq by storing table entry pointer into rseq_cs. */
 170                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 171                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
 172                 RSEQ_INJECT_ASM(3)
 173 #ifdef RSEQ_COMPARE_TWICE
 174                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 175 #endif
 176                 "ldr r0, %[v]\n\t"
 177                 "add r0, %[count]\n\t"
 178                 /* final store */
 179                 "str r0, %[v]\n\t"
 180                 "2:\n\t"
 181                 RSEQ_INJECT_ASM(4)
 182                 "b 5f\n\t"
 183                 RSEQ_ASM_DEFINE_ABORT(4, "", abort, 3, 1b, 2b, 4f)
 184                 "5:\n\t"
 185                 : /* gcc asm goto does not allow outputs */
 186                 : [cpu_id]              "r" (cpu),
 187                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_INDEX_CPU_ID_FIELD),
 188                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 189                   [v]                   "m" (*v),
 190                   [count]               "Ir" (count)
 191                   RSEQ_INJECT_INPUT
 192                 : "r0", "memory", "cc"
 193                   RSEQ_INJECT_CLOBBER
 194                 : abort
 195 #ifdef RSEQ_COMPARE_TWICE
 196                   , error1
 197 #endif
 198         );
 199         rseq_after_asm_goto();
 200         return 0;
 201 abort:
 202         rseq_after_asm_goto();
 203         RSEQ_INJECT_FAILED
 204         return -1;
 205 #ifdef RSEQ_COMPARE_TWICE
 206 error1:
 207         rseq_after_asm_goto();
 208         rseq_bug("cpu_id comparison failed");
 209 #endif
 210 }
 211
 212 static inline __attribute__((always_inline))
 213 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_load_cbne_store__ptr)(intptr_t *v, intptr_t expect,
 214                               intptr_t *v2, intptr_t expect2,
 215                               intptr_t newv, int cpu)
 216 {
 217         RSEQ_INJECT_C(9)
 218
 219         __asm__ __volatile__ goto (
 220                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
 221                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[ne])
 222 #ifdef RSEQ_COMPARE_TWICE
 223                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 224                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 225                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
 226 #endif
 227                 /* Start rseq by storing table entry pointer into rseq_cs. */
 228                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 229                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
 230                 RSEQ_INJECT_ASM(3)
 231                 "ldr r0, %[v]\n\t"
 232                 "cmp %[expect], r0\n\t"
 233                 "bne %l[ne]\n\t"
 234                 RSEQ_INJECT_ASM(4)
 235                 "ldr r0, %[v2]\n\t"
 236                 "cmp %[expect2], r0\n\t"
 237                 "bne %l[ne]\n\t"
 238                 RSEQ_INJECT_ASM(5)
 239 #ifdef RSEQ_COMPARE_TWICE
 240                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 241                 "ldr r0, %[v]\n\t"
 242                 "cmp %[expect], r0\n\t"
 243                 "bne %l[error2]\n\t"
 244                 "ldr r0, %[v2]\n\t"
 245                 "cmp %[expect2], r0\n\t"
 246                 "bne %l[error3]\n\t"
 247 #endif
 248                 /* final store */
 249                 "str %[newv], %[v]\n\t"
 250                 "2:\n\t"
 251                 RSEQ_INJECT_ASM(6)
 252                 "b 5f\n\t"
 253                 RSEQ_ASM_DEFINE_ABORT(4, "", abort, 3, 1b, 2b, 4f)
 254                 "5:\n\t"
 255                 : /* gcc asm goto does not allow outputs */
 256                 : [cpu_id]              "r" (cpu),
 257                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_INDEX_CPU_ID_FIELD),
 258                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 259                   /* cmp2 input */
 260                   [v2]                  "m" (*v2),
 261                   [expect2]             "r" (expect2),
 262                   /* final store input */
 263                   [v]                   "m" (*v),
 264                   [expect]              "r" (expect),
 265                   [newv]                "r" (newv)
 266                   RSEQ_INJECT_INPUT
 267                 : "r0", "memory", "cc"
 268                   RSEQ_INJECT_CLOBBER
 269                 : abort, ne
 270 #ifdef RSEQ_COMPARE_TWICE
 271                   , error1, error2, error3
 272 #endif
 273         );
 274         rseq_after_asm_goto();
 275         return 0;
 276 abort:
 277         rseq_after_asm_goto();
 278         RSEQ_INJECT_FAILED
 279         return -1;
 280 ne:
 281         rseq_after_asm_goto();
 282         return 1;
 283 #ifdef RSEQ_COMPARE_TWICE
 284 error1:
 285         rseq_after_asm_goto();
 286         rseq_bug("cpu_id comparison failed");
 287 error2:
 288         rseq_after_asm_goto();
 289         rseq_bug("1st expected value comparison failed");
 290 error3:
 291         rseq_after_asm_goto();
 292         rseq_bug("2nd expected value comparison failed");
 293 #endif
 294 }
 295
 296 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
 297         (defined(RSEQ_TEMPLATE_INDEX_CPU_ID) || defined(RSEQ_TEMPLATE_INDEX_MM_CID)) */
 298
 299 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
 300         (defined(RSEQ_TEMPLATE_INDEX_CPU_ID) || defined(RSEQ_TEMPLATE_INDEX_MM_CID))
 301
 302 static inline __attribute__((always_inline))
 303 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_store_store__ptr)(intptr_t *v, intptr_t expect,
 304                                  intptr_t *v2, intptr_t newv2,
 305                                  intptr_t newv, int cpu)
 306 {
 307         RSEQ_INJECT_C(9)
 308
 309         __asm__ __volatile__ goto (
 310                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
 311                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[ne])
 312 #ifdef RSEQ_COMPARE_TWICE
 313                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 314                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 315 #endif
 316                 /* Start rseq by storing table entry pointer into rseq_cs. */
 317                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 318                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
 319                 RSEQ_INJECT_ASM(3)
 320                 "ldr r0, %[v]\n\t"
 321                 "cmp %[expect], r0\n\t"
 322                 "bne %l[ne]\n\t"
 323                 RSEQ_INJECT_ASM(4)
 324 #ifdef RSEQ_COMPARE_TWICE
 325                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 326                 "ldr r0, %[v]\n\t"
 327                 "cmp %[expect], r0\n\t"
 328                 "bne %l[error2]\n\t"
 329 #endif
 330                 /* try store */
 331                 "str %[newv2], %[v2]\n\t"
 332                 RSEQ_INJECT_ASM(5)
 333 #ifdef RSEQ_TEMPLATE_MO_RELEASE
 334                 "dmb\n\t"       /* full mb provides store-release */
 335 #endif
 336                 /* final store */
 337                 "str %[newv], %[v]\n\t"
 338                 "2:\n\t"
 339                 RSEQ_INJECT_ASM(6)
 340                 "b 5f\n\t"
 341                 RSEQ_ASM_DEFINE_ABORT(4, "", abort, 3, 1b, 2b, 4f)
 342                 "5:\n\t"
 343                 : /* gcc asm goto does not allow outputs */
 344                 : [cpu_id]              "r" (cpu),
 345                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_INDEX_CPU_ID_FIELD),
 346                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 347                   /* try store input */
 348                   [v2]                  "m" (*v2),
 349                   [newv2]               "r" (newv2),
 350                   /* final store input */
 351                   [v]                   "m" (*v),
 352                   [expect]              "r" (expect),
 353                   [newv]                "r" (newv)
 354                   RSEQ_INJECT_INPUT
 355                 : "r0", "memory", "cc"
 356                   RSEQ_INJECT_CLOBBER
 357                 : abort, ne
 358 #ifdef RSEQ_COMPARE_TWICE
 359                   , error1, error2
 360 #endif
 361         );
 362         rseq_after_asm_goto();
 363         return 0;
 364 abort:
 365         rseq_after_asm_goto();
 366         RSEQ_INJECT_FAILED
 367         return -1;
 368 ne:
 369         rseq_after_asm_goto();
 370         return 1;
 371 #ifdef RSEQ_COMPARE_TWICE
 372 error1:
 373         rseq_after_asm_goto();
 374         rseq_bug("cpu_id comparison failed");
 375 error2:
 376         rseq_after_asm_goto();
 377         rseq_bug("expected value comparison failed");
 378 #endif
 379 }
 380
 381
 382 static inline __attribute__((always_inline))
 383 int RSEQ_TEMPLATE_IDENTIFIER(rseq_load_cbne_memcpy_store__ptr)(intptr_t *v, intptr_t expect,
 384                                  void *dst, void *src, size_t len,
 385                                  intptr_t newv, int cpu)
 386 {
 387         uint32_t rseq_scratch[3];
 388
 389         RSEQ_INJECT_C(9)
 390
 391         __asm__ __volatile__ goto (
 392                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
 393                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[ne])
 394 #ifdef RSEQ_COMPARE_TWICE
 395                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 396                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 397 #endif
 398                 "str %[src], %[rseq_scratch0]\n\t"
 399                 "str %[dst], %[rseq_scratch1]\n\t"
 400                 "str %[len], %[rseq_scratch2]\n\t"
 401                 /* Start rseq by storing table entry pointer into rseq_cs. */
 402                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 403                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
 404                 RSEQ_INJECT_ASM(3)
 405                 "ldr r0, %[v]\n\t"
 406                 "cmp %[expect], r0\n\t"
 407                 "bne 5f\n\t"
 408                 RSEQ_INJECT_ASM(4)
 409 #ifdef RSEQ_COMPARE_TWICE
 410                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 6f)
 411                 "ldr r0, %[v]\n\t"
 412                 "cmp %[expect], r0\n\t"
 413                 "bne 7f\n\t"
 414 #endif
 415                 /* try memcpy */
 416                 "cmp %[len], #0\n\t" \
 417                 "beq 333f\n\t" \
 418                 "222:\n\t" \
 419                 "ldrb %%r0, [%[src]]\n\t" \
 420                 "strb %%r0, [%[dst]]\n\t" \
 421                 "adds %[src], #1\n\t" \
 422                 "adds %[dst], #1\n\t" \
 423                 "subs %[len], #1\n\t" \
 424                 "bne 222b\n\t" \
 425                 "333:\n\t" \
 426                 RSEQ_INJECT_ASM(5)
 427 #ifdef RSEQ_TEMPLATE_MO_RELEASE
 428                 "dmb\n\t"       /* full mb provides store-release */
 429 #endif
 430                 /* final store */
 431                 "str %[newv], %[v]\n\t"
 432                 "2:\n\t"
 433                 RSEQ_INJECT_ASM(6)
 434                 /* teardown */
 435                 "ldr %[len], %[rseq_scratch2]\n\t"
 436                 "ldr %[dst], %[rseq_scratch1]\n\t"
 437                 "ldr %[src], %[rseq_scratch0]\n\t"
 438                 "b 8f\n\t"
 439                 RSEQ_ASM_DEFINE_ABORT(4,
 440                                       /* teardown */
 441                                       "ldr %[len], %[rseq_scratch2]\n\t"
 442                                       "ldr %[dst], %[rseq_scratch1]\n\t"
 443                                       "ldr %[src], %[rseq_scratch0]\n\t",
 444                                       abort, 3, 1b, 2b, 4f)
 445                 RSEQ_ASM_DEFINE_TEARDOWN(5,
 446                                         /* teardown */
 447                                         "ldr %[len], %[rseq_scratch2]\n\t"
 448                                         "ldr %[dst], %[rseq_scratch1]\n\t"
 449                                         "ldr %[src], %[rseq_scratch0]\n\t",
 450                                         ne)
 451 #ifdef RSEQ_COMPARE_TWICE
 452                 RSEQ_ASM_DEFINE_TEARDOWN(6,
 453                                         /* teardown */
 454                                         "ldr %[len], %[rseq_scratch2]\n\t"
 455                                         "ldr %[dst], %[rseq_scratch1]\n\t"
 456                                         "ldr %[src], %[rseq_scratch0]\n\t",
 457                                         error1)
 458                 RSEQ_ASM_DEFINE_TEARDOWN(7,
 459                                         /* teardown */
 460                                         "ldr %[len], %[rseq_scratch2]\n\t"
 461                                         "ldr %[dst], %[rseq_scratch1]\n\t"
 462                                         "ldr %[src], %[rseq_scratch0]\n\t",
 463                                         error2)
 464 #endif
 465                 "8:\n\t"
 466                 : /* gcc asm goto does not allow outputs */
 467                 : [cpu_id]              "r" (cpu),
 468                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_INDEX_CPU_ID_FIELD),
 469                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 470                   /* final store input */
 471                   [v]                   "m" (*v),
 472                   [expect]              "r" (expect),
 473                   [newv]                "r" (newv),
 474                   /* try memcpy input */
 475                   [dst]                 "r" (dst),
 476                   [src]                 "r" (src),
 477                   [len]                 "r" (len),
 478                   [rseq_scratch0]       "m" (rseq_scratch[0]),
 479                   [rseq_scratch1]       "m" (rseq_scratch[1]),
 480                   [rseq_scratch2]       "m" (rseq_scratch[2])
 481                   RSEQ_INJECT_INPUT
 482                 : "r0", "memory", "cc"
 483                   RSEQ_INJECT_CLOBBER
 484                 : abort, ne
 485 #ifdef RSEQ_COMPARE_TWICE
 486                   , error1, error2
 487 #endif
 488         );
 489         rseq_after_asm_goto();
 490         return 0;
 491 abort:
 492         rseq_after_asm_goto();
 493         RSEQ_INJECT_FAILED
 494         return -1;
 495 ne:
 496         rseq_after_asm_goto();
 497         return 1;
 498 #ifdef RSEQ_COMPARE_TWICE
 499 error1:
 500         rseq_after_asm_goto();
 501         rseq_bug("cpu_id comparison failed");
 502 error2:
 503         rseq_after_asm_goto();
 504         rseq_bug("expected value comparison failed");
 505 #endif
 506 }
 507
 508 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
 509         (defined(RSEQ_TEMPLATE_INDEX_CPU_ID) || defined(RSEQ_TEMPLATE_INDEX_MM_CID)) */
 510
 511 #include "rseq/arch/templates/bits-reset.h"