include/rseq/rseq-arm-bits.h

   1 /* SPDX-License-Identifier: MIT */
   2 /* SPDX-FileCopyrightText: 2016-2018 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
   3
   4 /*
   5  * rseq-arm-bits.h
   6  */
   7
   8 #include "rseq-bits-template.h"
   9
  10 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
  11         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
  12
  13 static inline __attribute__((always_inline))
  14 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
  15 {
  16         RSEQ_INJECT_C(9)
  17
  18         __asm__ __volatile__ goto (
  19                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
  20                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
  21 #ifdef RSEQ_COMPARE_TWICE
  22                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
  23                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
  24 #endif
  25                 /* Start rseq by storing table entry pointer into rseq_cs. */
  26                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
  27                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
  28                 RSEQ_INJECT_ASM(3)
  29                 "ldr r0, %[v]\n\t"
  30                 "cmp %[expect], r0\n\t"
  31                 "bne %l[cmpfail]\n\t"
  32                 RSEQ_INJECT_ASM(4)
  33 #ifdef RSEQ_COMPARE_TWICE
  34                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
  35                 "ldr r0, %[v]\n\t"
  36                 "cmp %[expect], r0\n\t"
  37                 "bne %l[error2]\n\t"
  38 #endif
  39                 /* final store */
  40                 "str %[newv], %[v]\n\t"
  41                 "2:\n\t"
  42                 RSEQ_INJECT_ASM(5)
  43                 "b 5f\n\t"
  44                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
  45                 "5:\n\t"
  46                 : /* gcc asm goto does not allow outputs */
  47                 : [cpu_id]              "r" (cpu),
  48                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
  49                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
  50                   [v]                   "m" (*v),
  51                   [expect]              "r" (expect),
  52                   [newv]                "r" (newv)
  53                   RSEQ_INJECT_INPUT
  54                 : "r0", "memory", "cc"
  55                   RSEQ_INJECT_CLOBBER
  56                 : abort, cmpfail
  57 #ifdef RSEQ_COMPARE_TWICE
  58                   , error1, error2
  59 #endif
  60         );
  61         rseq_after_asm_goto();
  62         return 0;
  63 abort:
  64         rseq_after_asm_goto();
  65         RSEQ_INJECT_FAILED
  66         return -1;
  67 cmpfail:
  68         rseq_after_asm_goto();
  69         return 1;
  70 #ifdef RSEQ_COMPARE_TWICE
  71 error1:
  72         rseq_after_asm_goto();
  73         rseq_bug("cpu_id comparison failed");
  74 error2:
  75         rseq_after_asm_goto();
  76         rseq_bug("expected value comparison failed");
  77 #endif
  78 }
  79
  80 static inline __attribute__((always_inline))
  81 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
  82                                long voffp, intptr_t *load, int cpu)
  83 {
  84         RSEQ_INJECT_C(9)
  85
  86         __asm__ __volatile__ goto (
  87                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
  88                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
  89 #ifdef RSEQ_COMPARE_TWICE
  90                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
  91                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
  92 #endif
  93                 /* Start rseq by storing table entry pointer into rseq_cs. */
  94                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
  95                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
  96                 RSEQ_INJECT_ASM(3)
  97                 "ldr r0, %[v]\n\t"
  98                 "cmp %[expectnot], r0\n\t"
  99                 "beq %l[cmpfail]\n\t"
 100                 RSEQ_INJECT_ASM(4)
 101 #ifdef RSEQ_COMPARE_TWICE
 102                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 103                 "ldr r0, %[v]\n\t"
 104                 "cmp %[expectnot], r0\n\t"
 105                 "beq %l[error2]\n\t"
 106 #endif
 107                 "str r0, %[load]\n\t"
 108                 "add r0, %[voffp]\n\t"
 109                 "ldr r0, [r0]\n\t"
 110                 /* final store */
 111                 "str r0, %[v]\n\t"
 112                 "2:\n\t"
 113                 RSEQ_INJECT_ASM(5)
 114                 "b 5f\n\t"
 115                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 116                 "5:\n\t"
 117                 : /* gcc asm goto does not allow outputs */
 118                 : [cpu_id]              "r" (cpu),
 119                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 120                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 121                   /* final store input */
 122                   [v]                   "m" (*v),
 123                   [expectnot]           "r" (expectnot),
 124                   [voffp]               "Ir" (voffp),
 125                   [load]                "m" (*load)
 126                   RSEQ_INJECT_INPUT
 127                 : "r0", "memory", "cc"
 128                   RSEQ_INJECT_CLOBBER
 129                 : abort, cmpfail
 130 #ifdef RSEQ_COMPARE_TWICE
 131                   , error1, error2
 132 #endif
 133         );
 134         rseq_after_asm_goto();
 135         return 0;
 136 abort:
 137         rseq_after_asm_goto();
 138         RSEQ_INJECT_FAILED
 139         return -1;
 140 cmpfail:
 141         rseq_after_asm_goto();
 142         return 1;
 143 #ifdef RSEQ_COMPARE_TWICE
 144 error1:
 145         rseq_after_asm_goto();
 146         rseq_bug("cpu_id comparison failed");
 147 error2:
 148         rseq_after_asm_goto();
 149         rseq_bug("expected value comparison failed");
 150 #endif
 151 }
 152
 153 static inline __attribute__((always_inline))
 154 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
 155 {
 156         RSEQ_INJECT_C(9)
 157
 158         __asm__ __volatile__ goto (
 159                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
 160 #ifdef RSEQ_COMPARE_TWICE
 161                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 162 #endif
 163                 /* Start rseq by storing table entry pointer into rseq_cs. */
 164                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 165                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
 166                 RSEQ_INJECT_ASM(3)
 167 #ifdef RSEQ_COMPARE_TWICE
 168                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 169 #endif
 170                 "ldr r0, %[v]\n\t"
 171                 "add r0, %[count]\n\t"
 172                 /* final store */
 173                 "str r0, %[v]\n\t"
 174                 "2:\n\t"
 175                 RSEQ_INJECT_ASM(4)
 176                 "b 5f\n\t"
 177                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 178                 "5:\n\t"
 179                 : /* gcc asm goto does not allow outputs */
 180                 : [cpu_id]              "r" (cpu),
 181                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 182                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 183                   [v]                   "m" (*v),
 184                   [count]               "Ir" (count)
 185                   RSEQ_INJECT_INPUT
 186                 : "r0", "memory", "cc"
 187                   RSEQ_INJECT_CLOBBER
 188                 : abort
 189 #ifdef RSEQ_COMPARE_TWICE
 190                   , error1
 191 #endif
 192         );
 193         rseq_after_asm_goto();
 194         return 0;
 195 abort:
 196         rseq_after_asm_goto();
 197         RSEQ_INJECT_FAILED
 198         return -1;
 199 #ifdef RSEQ_COMPARE_TWICE
 200 error1:
 201         rseq_after_asm_goto();
 202         rseq_bug("cpu_id comparison failed");
 203 #endif
 204 }
 205
 206 static inline __attribute__((always_inline))
 207 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
 208                               intptr_t *v2, intptr_t expect2,
 209                               intptr_t newv, int cpu)
 210 {
 211         RSEQ_INJECT_C(9)
 212
 213         __asm__ __volatile__ goto (
 214                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
 215                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 216 #ifdef RSEQ_COMPARE_TWICE
 217                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 218                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 219                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
 220 #endif
 221                 /* Start rseq by storing table entry pointer into rseq_cs. */
 222                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 223                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
 224                 RSEQ_INJECT_ASM(3)
 225                 "ldr r0, %[v]\n\t"
 226                 "cmp %[expect], r0\n\t"
 227                 "bne %l[cmpfail]\n\t"
 228                 RSEQ_INJECT_ASM(4)
 229                 "ldr r0, %[v2]\n\t"
 230                 "cmp %[expect2], r0\n\t"
 231                 "bne %l[cmpfail]\n\t"
 232                 RSEQ_INJECT_ASM(5)
 233 #ifdef RSEQ_COMPARE_TWICE
 234                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 235                 "ldr r0, %[v]\n\t"
 236                 "cmp %[expect], r0\n\t"
 237                 "bne %l[error2]\n\t"
 238                 "ldr r0, %[v2]\n\t"
 239                 "cmp %[expect2], r0\n\t"
 240                 "bne %l[error3]\n\t"
 241 #endif
 242                 /* final store */
 243                 "str %[newv], %[v]\n\t"
 244                 "2:\n\t"
 245                 RSEQ_INJECT_ASM(6)
 246                 "b 5f\n\t"
 247                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 248                 "5:\n\t"
 249                 : /* gcc asm goto does not allow outputs */
 250                 : [cpu_id]              "r" (cpu),
 251                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 252                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 253                   /* cmp2 input */
 254                   [v2]                  "m" (*v2),
 255                   [expect2]             "r" (expect2),
 256                   /* final store input */
 257                   [v]                   "m" (*v),
 258                   [expect]              "r" (expect),
 259                   [newv]                "r" (newv)
 260                   RSEQ_INJECT_INPUT
 261                 : "r0", "memory", "cc"
 262                   RSEQ_INJECT_CLOBBER
 263                 : abort, cmpfail
 264 #ifdef RSEQ_COMPARE_TWICE
 265                   , error1, error2, error3
 266 #endif
 267         );
 268         rseq_after_asm_goto();
 269         return 0;
 270 abort:
 271         rseq_after_asm_goto();
 272         RSEQ_INJECT_FAILED
 273         return -1;
 274 cmpfail:
 275         rseq_after_asm_goto();
 276         return 1;
 277 #ifdef RSEQ_COMPARE_TWICE
 278 error1:
 279         rseq_after_asm_goto();
 280         rseq_bug("cpu_id comparison failed");
 281 error2:
 282         rseq_after_asm_goto();
 283         rseq_bug("1st expected value comparison failed");
 284 error3:
 285         rseq_after_asm_goto();
 286         rseq_bug("2nd expected value comparison failed");
 287 #endif
 288 }
 289
 290 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
 291         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
 292
 293 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
 294         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
 295
 296 static inline __attribute__((always_inline))
 297 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
 298                                  intptr_t *v2, intptr_t newv2,
 299                                  intptr_t newv, int cpu)
 300 {
 301         RSEQ_INJECT_C(9)
 302
 303         __asm__ __volatile__ goto (
 304                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
 305                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 306 #ifdef RSEQ_COMPARE_TWICE
 307                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 308                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 309 #endif
 310                 /* Start rseq by storing table entry pointer into rseq_cs. */
 311                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 312                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
 313                 RSEQ_INJECT_ASM(3)
 314                 "ldr r0, %[v]\n\t"
 315                 "cmp %[expect], r0\n\t"
 316                 "bne %l[cmpfail]\n\t"
 317                 RSEQ_INJECT_ASM(4)
 318 #ifdef RSEQ_COMPARE_TWICE
 319                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 320                 "ldr r0, %[v]\n\t"
 321                 "cmp %[expect], r0\n\t"
 322                 "bne %l[error2]\n\t"
 323 #endif
 324                 /* try store */
 325                 "str %[newv2], %[v2]\n\t"
 326                 RSEQ_INJECT_ASM(5)
 327 #ifdef RSEQ_TEMPLATE_MO_RELEASE
 328                 "dmb\n\t"       /* full mb provides store-release */
 329 #endif
 330                 /* final store */
 331                 "str %[newv], %[v]\n\t"
 332                 "2:\n\t"
 333                 RSEQ_INJECT_ASM(6)
 334                 "b 5f\n\t"
 335                 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
 336                 "5:\n\t"
 337                 : /* gcc asm goto does not allow outputs */
 338                 : [cpu_id]              "r" (cpu),
 339                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 340                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 341                   /* try store input */
 342                   [v2]                  "m" (*v2),
 343                   [newv2]               "r" (newv2),
 344                   /* final store input */
 345                   [v]                   "m" (*v),
 346                   [expect]              "r" (expect),
 347                   [newv]                "r" (newv)
 348                   RSEQ_INJECT_INPUT
 349                 : "r0", "memory", "cc"
 350                   RSEQ_INJECT_CLOBBER
 351                 : abort, cmpfail
 352 #ifdef RSEQ_COMPARE_TWICE
 353                   , error1, error2
 354 #endif
 355         );
 356         rseq_after_asm_goto();
 357         return 0;
 358 abort:
 359         rseq_after_asm_goto();
 360         RSEQ_INJECT_FAILED
 361         return -1;
 362 cmpfail:
 363         rseq_after_asm_goto();
 364         return 1;
 365 #ifdef RSEQ_COMPARE_TWICE
 366 error1:
 367         rseq_after_asm_goto();
 368         rseq_bug("cpu_id comparison failed");
 369 error2:
 370         rseq_after_asm_goto();
 371         rseq_bug("expected value comparison failed");
 372 #endif
 373 }
 374
 375
 376 static inline __attribute__((always_inline))
 377 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
 378                                  void *dst, void *src, size_t len,
 379                                  intptr_t newv, int cpu)
 380 {
 381         uint32_t rseq_scratch[3];
 382
 383         RSEQ_INJECT_C(9)
 384
 385         __asm__ __volatile__ goto (
 386                 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
 387                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 388 #ifdef RSEQ_COMPARE_TWICE
 389                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 390                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 391 #endif
 392                 "str %[src], %[rseq_scratch0]\n\t"
 393                 "str %[dst], %[rseq_scratch1]\n\t"
 394                 "str %[len], %[rseq_scratch2]\n\t"
 395                 /* Start rseq by storing table entry pointer into rseq_cs. */
 396                 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
 397                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
 398                 RSEQ_INJECT_ASM(3)
 399                 "ldr r0, %[v]\n\t"
 400                 "cmp %[expect], r0\n\t"
 401                 "bne 5f\n\t"
 402                 RSEQ_INJECT_ASM(4)
 403 #ifdef RSEQ_COMPARE_TWICE
 404                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 6f)
 405                 "ldr r0, %[v]\n\t"
 406                 "cmp %[expect], r0\n\t"
 407                 "bne 7f\n\t"
 408 #endif
 409                 /* try memcpy */
 410                 "cmp %[len], #0\n\t" \
 411                 "beq 333f\n\t" \
 412                 "222:\n\t" \
 413                 "ldrb %%r0, [%[src]]\n\t" \
 414                 "strb %%r0, [%[dst]]\n\t" \
 415                 "adds %[src], #1\n\t" \
 416                 "adds %[dst], #1\n\t" \
 417                 "subs %[len], #1\n\t" \
 418                 "bne 222b\n\t" \
 419                 "333:\n\t" \
 420                 RSEQ_INJECT_ASM(5)
 421 #ifdef RSEQ_TEMPLATE_MO_RELEASE
 422                 "dmb\n\t"       /* full mb provides store-release */
 423 #endif
 424                 /* final store */
 425                 "str %[newv], %[v]\n\t"
 426                 "2:\n\t"
 427                 RSEQ_INJECT_ASM(6)
 428                 /* teardown */
 429                 "ldr %[len], %[rseq_scratch2]\n\t"
 430                 "ldr %[dst], %[rseq_scratch1]\n\t"
 431                 "ldr %[src], %[rseq_scratch0]\n\t"
 432                 "b 8f\n\t"
 433                 RSEQ_ASM_DEFINE_ABORT(3, 4,
 434                                       /* teardown */
 435                                       "ldr %[len], %[rseq_scratch2]\n\t"
 436                                       "ldr %[dst], %[rseq_scratch1]\n\t"
 437                                       "ldr %[src], %[rseq_scratch0]\n\t",
 438                                       abort, 1b, 2b, 4f)
 439                 RSEQ_ASM_DEFINE_CMPFAIL(5,
 440                                         /* teardown */
 441                                         "ldr %[len], %[rseq_scratch2]\n\t"
 442                                         "ldr %[dst], %[rseq_scratch1]\n\t"
 443                                         "ldr %[src], %[rseq_scratch0]\n\t",
 444                                         cmpfail)
 445 #ifdef RSEQ_COMPARE_TWICE
 446                 RSEQ_ASM_DEFINE_CMPFAIL(6,
 447                                         /* teardown */
 448                                         "ldr %[len], %[rseq_scratch2]\n\t"
 449                                         "ldr %[dst], %[rseq_scratch1]\n\t"
 450                                         "ldr %[src], %[rseq_scratch0]\n\t",
 451                                         error1)
 452                 RSEQ_ASM_DEFINE_CMPFAIL(7,
 453                                         /* teardown */
 454                                         "ldr %[len], %[rseq_scratch2]\n\t"
 455                                         "ldr %[dst], %[rseq_scratch1]\n\t"
 456                                         "ldr %[src], %[rseq_scratch0]\n\t",
 457                                         error2)
 458 #endif
 459                 "8:\n\t"
 460                 : /* gcc asm goto does not allow outputs */
 461                 : [cpu_id]              "r" (cpu),
 462                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 463                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 464                   /* final store input */
 465                   [v]                   "m" (*v),
 466                   [expect]              "r" (expect),
 467                   [newv]                "r" (newv),
 468                   /* try memcpy input */
 469                   [dst]                 "r" (dst),
 470                   [src]                 "r" (src),
 471                   [len]                 "r" (len),
 472                   [rseq_scratch0]       "m" (rseq_scratch[0]),
 473                   [rseq_scratch1]       "m" (rseq_scratch[1]),
 474                   [rseq_scratch2]       "m" (rseq_scratch[2])
 475                   RSEQ_INJECT_INPUT
 476                 : "r0", "memory", "cc"
 477                   RSEQ_INJECT_CLOBBER
 478                 : abort, cmpfail
 479 #ifdef RSEQ_COMPARE_TWICE
 480                   , error1, error2
 481 #endif
 482         );
 483         rseq_after_asm_goto();
 484         return 0;
 485 abort:
 486         rseq_after_asm_goto();
 487         RSEQ_INJECT_FAILED
 488         return -1;
 489 cmpfail:
 490         rseq_after_asm_goto();
 491         return 1;
 492 #ifdef RSEQ_COMPARE_TWICE
 493 error1:
 494         rseq_after_asm_goto();
 495         rseq_bug("cpu_id comparison failed");
 496 error2:
 497         rseq_after_asm_goto();
 498         rseq_bug("expected value comparison failed");
 499 #endif
 500 }
 501
 502 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
 503         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
 504
 505 #include "rseq-bits-reset.h"