include/rseq/rseq-s390-bits.h

   1 /* SPDX-License-Identifier: MIT */
   2 /* SPDX-FileCopyrightText: 2018 Vasily Gorbik <gor@linux.ibm.com> */
   3
   4 #include "rseq-bits-template.h"
   5
   6 /*
   7  * Refer to rseq-pseudocode.h for documentation and pseudo-code of the
   8  * rseq critical section helpers.
   9  */
  10 #include "rseq-pseudocode.h"
  11
  12 #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
  13         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
  14
  15 static inline __attribute__((always_inline))
  16 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
  17 {
  18         RSEQ_INJECT_C(9)
  19
  20         __asm__ __volatile__ goto (
  21                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
  22                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
  23 #ifdef RSEQ_COMPARE_TWICE
  24                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
  25                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
  26 #endif
  27                 /* Start rseq by storing table entry pointer into rseq_cs. */
  28                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
  29                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
  30                 RSEQ_INJECT_ASM(3)
  31                 LONG_CMP " %[expect], %[v]\n\t"
  32                 "jnz %l[cmpfail]\n\t"
  33                 RSEQ_INJECT_ASM(4)
  34 #ifdef RSEQ_COMPARE_TWICE
  35                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
  36                 LONG_CMP " %[expect], %[v]\n\t"
  37                 "jnz %l[error2]\n\t"
  38 #endif
  39                 /* final store */
  40                 LONG_S " %[newv], %[v]\n\t"
  41                 "2:\n\t"
  42                 RSEQ_INJECT_ASM(5)
  43                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
  44                 : /* gcc asm goto does not allow outputs */
  45                 : [cpu_id]              "r" (cpu),
  46                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
  47                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
  48                   [v]                   "m" (*v),
  49                   [expect]              "r" (expect),
  50                   [newv]                "r" (newv)
  51                   RSEQ_INJECT_INPUT
  52                 : "memory", "cc", "r0"
  53                   RSEQ_INJECT_CLOBBER
  54                 : abort, cmpfail
  55 #ifdef RSEQ_COMPARE_TWICE
  56                   , error1, error2
  57 #endif
  58         );
  59         rseq_after_asm_goto();
  60         return 0;
  61 abort:
  62         rseq_after_asm_goto();
  63         RSEQ_INJECT_FAILED
  64         return -1;
  65 cmpfail:
  66         rseq_after_asm_goto();
  67         return 1;
  68 #ifdef RSEQ_COMPARE_TWICE
  69 error1:
  70         rseq_after_asm_goto();
  71         rseq_bug("cpu_id comparison failed");
  72 error2:
  73         rseq_after_asm_goto();
  74         rseq_bug("expected value comparison failed");
  75 #endif
  76 }
  77
  78 /*
  79  * Compare @v against @expectnot. When it does _not_ match, load @v
  80  * into @load, and store the content of *@v + voffp into @v.
  81  */
  82 static inline __attribute__((always_inline))
  83 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
  84                                long voffp, intptr_t *load, int cpu)
  85 {
  86         RSEQ_INJECT_C(9)
  87
  88         __asm__ __volatile__ goto (
  89                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
  90                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
  91 #ifdef RSEQ_COMPARE_TWICE
  92                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
  93                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
  94 #endif
  95                 /* Start rseq by storing table entry pointer into rseq_cs. */
  96                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
  97                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
  98                 RSEQ_INJECT_ASM(3)
  99                 LONG_L " %%r1, %[v]\n\t"
 100                 LONG_CMP_R " %%r1, %[expectnot]\n\t"
 101                 "je %l[cmpfail]\n\t"
 102                 RSEQ_INJECT_ASM(4)
 103 #ifdef RSEQ_COMPARE_TWICE
 104                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 105                 LONG_L " %%r1, %[v]\n\t"
 106                 LONG_CMP_R " %%r1, %[expectnot]\n\t"
 107                 "je %l[error2]\n\t"
 108 #endif
 109                 LONG_S " %%r1, %[load]\n\t"
 110                 LONG_ADD_R " %%r1, %[voffp]\n\t"
 111                 LONG_L " %%r1, 0(%%r1)\n\t"
 112                 /* final store */
 113                 LONG_S " %%r1, %[v]\n\t"
 114                 "2:\n\t"
 115                 RSEQ_INJECT_ASM(5)
 116                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 117                 : /* gcc asm goto does not allow outputs */
 118                 : [cpu_id]              "r" (cpu),
 119                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 120                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 121                   /* final store input */
 122                   [v]                   "m" (*v),
 123                   [expectnot]           "r" (expectnot),
 124                   [voffp]               "r" (voffp),
 125                   [load]                "m" (*load)
 126                   RSEQ_INJECT_INPUT
 127                 : "memory", "cc", "r0", "r1"
 128                   RSEQ_INJECT_CLOBBER
 129                 : abort, cmpfail
 130 #ifdef RSEQ_COMPARE_TWICE
 131                   , error1, error2
 132 #endif
 133         );
 134         rseq_after_asm_goto();
 135         return 0;
 136 abort:
 137         rseq_after_asm_goto();
 138         RSEQ_INJECT_FAILED
 139         return -1;
 140 cmpfail:
 141         rseq_after_asm_goto();
 142         return 1;
 143 #ifdef RSEQ_COMPARE_TWICE
 144 error1:
 145         rseq_after_asm_goto();
 146         rseq_bug("cpu_id comparison failed");
 147 error2:
 148         rseq_after_asm_goto();
 149         rseq_bug("expected value comparison failed");
 150 #endif
 151 }
 152
 153 static inline __attribute__((always_inline))
 154 int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
 155 {
 156         RSEQ_INJECT_C(9)
 157
 158         __asm__ __volatile__ goto (
 159                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 160 #ifdef RSEQ_COMPARE_TWICE
 161                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 162 #endif
 163                 /* Start rseq by storing table entry pointer into rseq_cs. */
 164                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 165                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
 166                 RSEQ_INJECT_ASM(3)
 167 #ifdef RSEQ_COMPARE_TWICE
 168                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 169 #endif
 170                 LONG_L " %%r0, %[v]\n\t"
 171                 LONG_ADD_R " %%r0, %[count]\n\t"
 172                 /* final store */
 173                 LONG_S " %%r0, %[v]\n\t"
 174                 "2:\n\t"
 175                 RSEQ_INJECT_ASM(4)
 176                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 177                 : /* gcc asm goto does not allow outputs */
 178                 : [cpu_id]              "r" (cpu),
 179                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 180                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 181                   /* final store input */
 182                   [v]                   "m" (*v),
 183                   [count]               "r" (count)
 184                   RSEQ_INJECT_INPUT
 185                 : "memory", "cc", "r0"
 186                   RSEQ_INJECT_CLOBBER
 187                 : abort
 188 #ifdef RSEQ_COMPARE_TWICE
 189                   , error1
 190 #endif
 191         );
 192         rseq_after_asm_goto();
 193         return 0;
 194 abort:
 195         rseq_after_asm_goto();
 196         RSEQ_INJECT_FAILED
 197         return -1;
 198 #ifdef RSEQ_COMPARE_TWICE
 199 error1:
 200         rseq_after_asm_goto();
 201         rseq_bug("cpu_id comparison failed");
 202 #endif
 203 }
 204
 205 static inline __attribute__((always_inline))
 206 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
 207                               intptr_t *v2, intptr_t expect2,
 208                               intptr_t newv, int cpu)
 209 {
 210         RSEQ_INJECT_C(9)
 211
 212         __asm__ __volatile__ goto (
 213                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 214                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 215 #ifdef RSEQ_COMPARE_TWICE
 216                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 217                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 218                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
 219 #endif
 220                 /* Start rseq by storing table entry pointer into rseq_cs. */
 221                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 222                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
 223                 RSEQ_INJECT_ASM(3)
 224                 LONG_CMP " %[expect], %[v]\n\t"
 225                 "jnz %l[cmpfail]\n\t"
 226                 RSEQ_INJECT_ASM(4)
 227                 LONG_CMP " %[expect2], %[v2]\n\t"
 228                 "jnz %l[cmpfail]\n\t"
 229                 RSEQ_INJECT_ASM(5)
 230 #ifdef RSEQ_COMPARE_TWICE
 231                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 232                 LONG_CMP " %[expect], %[v]\n\t"
 233                 "jnz %l[error2]\n\t"
 234                 LONG_CMP " %[expect2], %[v2]\n\t"
 235                 "jnz %l[error3]\n\t"
 236 #endif
 237                 /* final store */
 238                 LONG_S " %[newv], %[v]\n\t"
 239                 "2:\n\t"
 240                 RSEQ_INJECT_ASM(6)
 241                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 242                 : /* gcc asm goto does not allow outputs */
 243                 : [cpu_id]              "r" (cpu),
 244                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 245                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 246                   /* cmp2 input */
 247                   [v2]                  "m" (*v2),
 248                   [expect2]             "r" (expect2),
 249                   /* final store input */
 250                   [v]                   "m" (*v),
 251                   [expect]              "r" (expect),
 252                   [newv]                "r" (newv)
 253                   RSEQ_INJECT_INPUT
 254                 : "memory", "cc", "r0"
 255                   RSEQ_INJECT_CLOBBER
 256                 : abort, cmpfail
 257 #ifdef RSEQ_COMPARE_TWICE
 258                   , error1, error2, error3
 259 #endif
 260         );
 261         rseq_after_asm_goto();
 262         return 0;
 263 abort:
 264         rseq_after_asm_goto();
 265         RSEQ_INJECT_FAILED
 266         return -1;
 267 cmpfail:
 268         rseq_after_asm_goto();
 269         return 1;
 270 #ifdef RSEQ_COMPARE_TWICE
 271 error1:
 272         rseq_after_asm_goto();
 273         rseq_bug("cpu_id comparison failed");
 274 error2:
 275         rseq_after_asm_goto();
 276         rseq_bug("1st expected value comparison failed");
 277 error3:
 278         rseq_after_asm_goto();
 279         rseq_bug("2nd expected value comparison failed");
 280 #endif
 281 }
 282
 283 #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
 284         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
 285
 286 #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
 287         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
 288
 289 /* s390 is TSO. */
 290 static inline __attribute__((always_inline))
 291 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
 292                                  intptr_t *v2, intptr_t newv2,
 293                                  intptr_t newv, int cpu)
 294 {
 295         RSEQ_INJECT_C(9)
 296
 297         __asm__ __volatile__ goto (
 298                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 299                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 300 #ifdef RSEQ_COMPARE_TWICE
 301                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 302                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 303 #endif
 304                 /* Start rseq by storing table entry pointer into rseq_cs. */
 305                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 306                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
 307                 RSEQ_INJECT_ASM(3)
 308                 LONG_CMP " %[expect], %[v]\n\t"
 309                 "jnz %l[cmpfail]\n\t"
 310                 RSEQ_INJECT_ASM(4)
 311 #ifdef RSEQ_COMPARE_TWICE
 312                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, %l[error1])
 313                 LONG_CMP " %[expect], %[v]\n\t"
 314                 "jnz %l[error2]\n\t"
 315 #endif
 316                 /* try store */
 317                 LONG_S " %[newv2], %[v2]\n\t"
 318                 RSEQ_INJECT_ASM(5)
 319                 /* final store */
 320                 LONG_S " %[newv], %[v]\n\t"
 321                 "2:\n\t"
 322                 RSEQ_INJECT_ASM(6)
 323                 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
 324                 : /* gcc asm goto does not allow outputs */
 325                 : [cpu_id]              "r" (cpu),
 326                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 327                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 328                   /* try store input */
 329                   [v2]                  "m" (*v2),
 330                   [newv2]               "r" (newv2),
 331                   /* final store input */
 332                   [v]                   "m" (*v),
 333                   [expect]              "r" (expect),
 334                   [newv]                "r" (newv)
 335                   RSEQ_INJECT_INPUT
 336                 : "memory", "cc", "r0"
 337                   RSEQ_INJECT_CLOBBER
 338                 : abort, cmpfail
 339 #ifdef RSEQ_COMPARE_TWICE
 340                   , error1, error2
 341 #endif
 342         );
 343         rseq_after_asm_goto();
 344         return 0;
 345 abort:
 346         rseq_after_asm_goto();
 347         RSEQ_INJECT_FAILED
 348         return -1;
 349 cmpfail:
 350         rseq_after_asm_goto();
 351         return 1;
 352 #ifdef RSEQ_COMPARE_TWICE
 353 error1:
 354         rseq_after_asm_goto();
 355         rseq_bug("cpu_id comparison failed");
 356 error2:
 357         rseq_after_asm_goto();
 358         rseq_bug("expected value comparison failed");
 359 #endif
 360 }
 361
 362 /* s390 is TSO. */
 363 static inline __attribute__((always_inline))
 364 int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
 365                                  void *dst, void *src, size_t len,
 366                                  intptr_t newv, int cpu)
 367 {
 368         uint64_t rseq_scratch[3];
 369
 370         RSEQ_INJECT_C(9)
 371
 372         __asm__ __volatile__ goto (
 373                 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
 374                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
 375 #ifdef RSEQ_COMPARE_TWICE
 376                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
 377                 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
 378 #endif
 379                 LONG_S " %[src], %[rseq_scratch0]\n\t"
 380                 LONG_S " %[dst], %[rseq_scratch1]\n\t"
 381                 LONG_S " %[len], %[rseq_scratch2]\n\t"
 382                 /* Start rseq by storing table entry pointer into rseq_cs. */
 383                 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
 384                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 4f)
 385                 RSEQ_INJECT_ASM(3)
 386                 LONG_CMP " %[expect], %[v]\n\t"
 387                 "jnz 5f\n\t"
 388                 RSEQ_INJECT_ASM(4)
 389 #ifdef RSEQ_COMPARE_TWICE
 390                 RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, 6f)
 391                 LONG_CMP " %[expect], %[v]\n\t"
 392                 "jnz 7f\n\t"
 393 #endif
 394                 /* try memcpy */
 395                 LONG_LT_R " %[len], %[len]\n\t"
 396                 "jz 333f\n\t"
 397                 "222:\n\t"
 398                 "ic %%r0,0(%[src])\n\t"
 399                 "stc %%r0,0(%[dst])\n\t"
 400                 LONG_ADDI " %[src], 1\n\t"
 401                 LONG_ADDI " %[dst], 1\n\t"
 402                 LONG_ADDI " %[len], -1\n\t"
 403                 "jnz 222b\n\t"
 404                 "333:\n\t"
 405                 RSEQ_INJECT_ASM(5)
 406                 /* final store */
 407                 LONG_S " %[newv], %[v]\n\t"
 408                 "2:\n\t"
 409                 RSEQ_INJECT_ASM(6)
 410                 /* teardown */
 411                 LONG_L " %[len], %[rseq_scratch2]\n\t"
 412                 LONG_L " %[dst], %[rseq_scratch1]\n\t"
 413                 LONG_L " %[src], %[rseq_scratch0]\n\t"
 414                 RSEQ_ASM_DEFINE_ABORT(4,
 415                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 416                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 417                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 418                         abort)
 419                 RSEQ_ASM_DEFINE_CMPFAIL(5,
 420                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 421                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 422                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 423                         cmpfail)
 424 #ifdef RSEQ_COMPARE_TWICE
 425                 RSEQ_ASM_DEFINE_CMPFAIL(6,
 426                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 427                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 428                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 429                         error1)
 430                 RSEQ_ASM_DEFINE_CMPFAIL(7,
 431                         LONG_L " %[len], %[rseq_scratch2]\n\t"
 432                         LONG_L " %[dst], %[rseq_scratch1]\n\t"
 433                         LONG_L " %[src], %[rseq_scratch0]\n\t",
 434                         error2)
 435 #endif
 436                 : /* gcc asm goto does not allow outputs */
 437                 : [cpu_id]              "r" (cpu),
 438                   [current_cpu_id]      "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
 439                   [rseq_cs]             "m" (rseq_get_abi()->rseq_cs.arch.ptr),
 440                   /* final store input */
 441                   [v]                   "m" (*v),
 442                   [expect]              "r" (expect),
 443                   [newv]                "r" (newv),
 444                   /* try memcpy input */
 445                   [dst]                 "r" (dst),
 446                   [src]                 "r" (src),
 447                   [len]                 "r" (len),
 448                   [rseq_scratch0]       "m" (rseq_scratch[0]),
 449                   [rseq_scratch1]       "m" (rseq_scratch[1]),
 450                   [rseq_scratch2]       "m" (rseq_scratch[2])
 451                   RSEQ_INJECT_INPUT
 452                 : "memory", "cc", "r0"
 453                   RSEQ_INJECT_CLOBBER
 454                 : abort, cmpfail
 455 #ifdef RSEQ_COMPARE_TWICE
 456                   , error1, error2
 457 #endif
 458         );
 459         rseq_after_asm_goto();
 460         return 0;
 461 abort:
 462         rseq_after_asm_goto();
 463         RSEQ_INJECT_FAILED
 464         return -1;
 465 cmpfail:
 466         rseq_after_asm_goto();
 467         return 1;
 468 #ifdef RSEQ_COMPARE_TWICE
 469 error1:
 470         rseq_after_asm_goto();
 471         rseq_bug("cpu_id comparison failed");
 472 error2:
 473         rseq_after_asm_goto();
 474         rseq_bug("expected value comparison failed");
 475 #endif
 476 }
 477
 478 #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
 479         (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
 480
 481 #include "rseq-bits-reset.h"