include/rseq/rseq-s390.h

   1 /* SPDX-License-Identifier: MIT */
   2 /* SPDX-FileCopyrightText: 2018 Vasily Gorbik <gor@linux.ibm.com> */
   3 /* SPDX-FileCopyrightText: 2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
   4
   5 /*
   6  * rseq-s390.h
   7  */
   8
   9 /*
  10  * RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those
  11  * are not part of the public API.
  12  */
  13
  14 /*
  15  * RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the
  16  * access-register mode nor the linkage stack this instruction will always
  17  * cause a special-operation exception (the trap-enabled bit in the DUCT
  18  * is and will stay 0). The instruction pattern is
  19  *      b2 ff 0f ff     trap4   4095(%r0)
  20  */
  21 #define RSEQ_SIG        0xB2FF0FFF
  22
  23 /*
  24  * Refer to the Linux kernel memory model (LKMM) for documentation of
  25  * the memory barriers.
  26  */
  27
  28 /* CPU memory barrier. */
  29 #define rseq_smp_mb()   __asm__ __volatile__ ("bcr 15,0" ::: "memory")
  30 /* CPU read memory barrier */
  31 #define rseq_smp_rmb()  rseq_smp_mb()
  32 /* CPU write memory barrier */
  33 #define rseq_smp_wmb()  rseq_smp_mb()
  34
  35 /* Acquire: One-way permeable barrier. */
  36 #define rseq_smp_load_acquire(p)                                        \
  37 __extension__ ({                                                        \
  38         rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p));  \
  39         rseq_barrier();                                                 \
  40         ____p1;                                                         \
  41 })
  42
  43 /* Acquire barrier after control dependency. */
  44 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  45
  46 /* Release: One-way permeable barrier. */
  47 #define rseq_smp_store_release(p, v)                                    \
  48 do {                                                                    \
  49         rseq_barrier();                                                 \
  50         RSEQ_WRITE_ONCE(*(p), v);                                       \
  51 } while (0)
  52
  53 #ifdef __s390x__
  54
  55 /*
  56  * Helper macros to access a variable of pointer type stored in a 64-bit
  57  * integer. Only used internally in rseq headers.
  58  */
  59 #define RSEQ_ASM_LONG_L                 "lg"
  60 #define RSEQ_ASM_LONG_S                 "stg"
  61 #define RSEQ_ASM_LONG_LT_R              "ltgr"
  62 #define RSEQ_ASM_LONG_CMP               "cg"
  63 #define RSEQ_ASM_LONG_CMP_R             "cgr"
  64 #define RSEQ_ASM_LONG_ADDI              "aghi"
  65 #define RSEQ_ASM_LONG_ADD_R             "agr"
  66
  67 /* Only used in RSEQ_ASM_DEFINE_TABLE. */
  68 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
  69                                 start_ip, post_commit_offset, abort_ip) \
  70                 ".pushsection __rseq_cs, \"aw\"\n\t"                    \
  71                 ".balign 32\n\t"                                        \
  72                 __rseq_str(label) ":\n\t"                               \
  73                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  74                 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
  75                 ".popsection\n\t"                                       \
  76                 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
  77                 ".quad " __rseq_str(label) "b\n\t"                      \
  78                 ".popsection\n\t"
  79
  80 /*
  81  * Define the @exit_ip pointer as an exit point for the sequence of consecutive
  82  * assembly instructions at @start_ip.
  83  *
  84  *  @start_ip:
  85  *    Pointer to the first instruction of the sequence of consecutive assembly
  86  *    instructions.
  87  *  @exit_ip:
  88  *    Pointer to an exit point instruction.
  89  *
  90  * Exit points of a rseq critical section consist of all instructions outside
  91  * of the critical section where a critical section can either branch to or
  92  * reach through the normal course of its execution. The abort IP and the
  93  * post-commit IP are already part of the __rseq_cs section and should not be
  94  * explicitly defined as additional exit points. Knowing all exit points is
  95  * useful to assist debuggers stepping over the critical section.
  96  */
  97 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
  98                 ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
  99                 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
 100                 ".popsection\n\t"
 101
 102 #elif __s390__
 103
 104 /*
 105  * Helper macros to access a variable of pointer type stored in a 64-bit
 106  * integer. Only used internally in rseq headers.
 107  */
 108 #define RSEQ_ASM_LONG_L                 "l"
 109 #define RSEQ_ASM_LONG_S                 "st"
 110 #define RSEQ_ASM_LONG_LT_R              "ltr"
 111 #define RSEQ_ASM_LONG_CMP               "c"
 112 #define RSEQ_ASM_LONG_CMP_R             "cr"
 113 #define RSEQ_ASM_LONG_ADDI              "ahi"
 114 #define RSEQ_ASM_LONG_ADD_R             "ar"
 115
 116 /* Only used in RSEQ_ASM_DEFINE_TABLE. */
 117 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
 118                                 start_ip, post_commit_offset, abort_ip) \
 119                 ".pushsection __rseq_cs, \"aw\"\n\t"                    \
 120                 ".balign 32\n\t"                                        \
 121                 __rseq_str(label) ":\n\t"                               \
 122                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
 123                 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
 124                 ".popsection\n\t"                                       \
 125                 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
 126                 ".long 0x0, " __rseq_str(label) "b\n\t"                 \
 127                 ".popsection\n\t"
 128
 129 /*
 130  * Define the @exit_ip pointer as an exit point for the sequence of consecutive
 131  * assembly instructions at @start_ip.
 132  *
 133  *  @start_ip:
 134  *    Pointer to the first instruction of the sequence of consecutive assembly
 135  *    instructions.
 136  *  @exit_ip:
 137  *    Pointer to an exit point instruction.
 138  *
 139  * Exit points of a rseq critical section consist of all instructions outside
 140  * of the critical section where a critical section can either branch to or
 141  * reach through the normal course of its execution. The abort IP and the
 142  * post-commit IP are already part of the __rseq_cs section and should not be
 143  * explicitly defined as additional exit points. Knowing all exit points is
 144  * useful to assist debuggers stepping over the critical section.
 145  */
 146 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
 147                 ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
 148                 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
 149                 ".popsection\n\t"
 150
 151 #endif
 152
 153 /*
 154  * Define an rseq critical section structure of version 0 with no flags.
 155  *
 156  *  @label:
 157  *    Local label for the beginning of the critical section descriptor
 158  *    structure.
 159  *  @start_ip:
 160  *    Pointer to the first instruction of the sequence of consecutive assembly
 161  *    instructions.
 162  *  @post_commit_ip:
 163  *    Pointer to the instruction after the last instruction of the sequence of
 164  *    consecutive assembly instructions.
 165  *  @abort_ip:
 166  *    Pointer to the instruction where to move the execution flow in case of
 167  *    abort of the sequence of consecutive assembly instructions.
 168  */
 169 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
 170         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
 171                                 (post_commit_ip) - (start_ip), abort_ip)
 172
 173 /*
 174  * Define a critical section abort handler.
 175  *
 176  *  @label:
 177  *    Local label to the abort handler.
 178  *  @teardown:
 179  *    Sequence of instructions to run on abort.
 180  *  @abort_label:
 181  *    C label to jump to at the end of the sequence.
 182  */
 183 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
 184                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 185                 ".long " __rseq_str(RSEQ_SIG) "\n\t"                    \
 186                 __rseq_str(label) ":\n\t"                               \
 187                 teardown                                                \
 188                 "jg %l[" __rseq_str(abort_label) "]\n\t"                \
 189                 ".popsection\n\t"
 190
 191 /*
 192  * Define a critical section teardown handler.
 193  *
 194  *  @label:
 195  *    Local label to the teardown handler.
 196  *  @teardown:
 197  *    Sequence of instructions to run on teardown.
 198  *  @target_label:
 199  *    C label to jump to at the end of the sequence.
 200  */
 201 #define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label)         \
 202                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 203                 __rseq_str(label) ":\n\t"                               \
 204                 teardown                                                \
 205                 "jg %l[" __rseq_str(target_label) "]\n\t"               \
 206                 ".popsection\n\t"
 207
 208 /*
 209  * Store the address of the critical section descriptor structure at
 210  * @cs_label into the @rseq_cs pointer and emit the label @label, which
 211  * is the beginning of the sequence of consecutive assembly instructions.
 212  *
 213  *  @label:
 214  *    Local label to the beginning of the sequence of consecutive assembly
 215  *    instructions.
 216  *  @cs_label:
 217  *    Source local label to the critical section descriptor structure.
 218  *  @rseq_cs:
 219  *    Destination pointer where to store the address of the critical
 220  *    section descriptor structure.
 221  */
 222 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
 223                 RSEQ_INJECT_ASM(1)                                      \
 224                 "larl %%r0, " __rseq_str(cs_label) "\n\t"               \
 225                 RSEQ_ASM_LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t" \
 226                 __rseq_str(label) ":\n\t"
 227
 228 /* Jump to local label @label when @cpu_id != @current_cpu_id. */
 229 #define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label)             \
 230                 RSEQ_INJECT_ASM(2)                                      \
 231                 "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
 232                 "jnz " __rseq_str(label) "\n\t"
 233
 234 /* Per-cpu-id indexing. */
 235
 236 #define RSEQ_TEMPLATE_INDEX_CPU_ID
 237 #define RSEQ_TEMPLATE_MO_RELAXED
 238 #include "rseq-s390-bits.h"
 239 #undef RSEQ_TEMPLATE_MO_RELAXED
 240
 241 #define RSEQ_TEMPLATE_MO_RELEASE
 242 #include "rseq-s390-bits.h"
 243 #undef RSEQ_TEMPLATE_MO_RELEASE
 244 #undef RSEQ_TEMPLATE_INDEX_CPU_ID
 245
 246 /* Per-mm-cid indexing. */
 247
 248 #define RSEQ_TEMPLATE_INDEX_MM_CID
 249 #define RSEQ_TEMPLATE_MO_RELAXED
 250 #include "rseq-s390-bits.h"
 251 #undef RSEQ_TEMPLATE_MO_RELAXED
 252
 253 #define RSEQ_TEMPLATE_MO_RELEASE
 254 #include "rseq-s390-bits.h"
 255 #undef RSEQ_TEMPLATE_MO_RELEASE
 256 #undef RSEQ_TEMPLATE_INDEX_MM_CID
 257
 258 /* APIs which are not indexed. */
 259
 260 #define RSEQ_TEMPLATE_INDEX_NONE
 261 #define RSEQ_TEMPLATE_MO_RELAXED
 262 #include "rseq-s390-bits.h"
 263 #undef RSEQ_TEMPLATE_MO_RELAXED
 264 #undef RSEQ_TEMPLATE_INDEX_NONE