include/rseq/arch/s390.h

   1 /* SPDX-License-Identifier: MIT */
   2 /* SPDX-FileCopyrightText: 2018 Vasily Gorbik <gor@linux.ibm.com> */
   3 /* SPDX-FileCopyrightText: 2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
   4
   5 /*
   6  * rseq-s390.h
   7  */
   8
   9 /*
  10  * RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those
  11  * are not part of the public API.
  12  */
  13
  14 #ifndef _RSEQ_RSEQ_H
  15 #error "Never use <rseq/arch/s390.h> directly; include <rseq/rseq.h> instead."
  16 #endif
  17
  18 /*
  19  * RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the
  20  * access-register mode nor the linkage stack this instruction will always
  21  * cause a special-operation exception (the trap-enabled bit in the DUCT
  22  * is and will stay 0). The instruction pattern is
  23  *      b2 ff 0f ff     trap4   4095(%r0)
  24  */
  25 #define RSEQ_SIG        0xB2FF0FFF
  26
  27 /*
  28  * Refer to the Linux kernel memory model (LKMM) for documentation of
  29  * the memory barriers.
  30  */
  31
  32 /* CPU memory barrier. */
  33 #define rseq_smp_mb()   __asm__ __volatile__ ("bcr 15,0" ::: "memory")
  34 /* CPU read memory barrier */
  35 #define rseq_smp_rmb()  rseq_smp_mb()
  36 /* CPU write memory barrier */
  37 #define rseq_smp_wmb()  rseq_smp_mb()
  38
  39 /* Acquire: One-way permeable barrier. */
  40 #define rseq_smp_load_acquire(p)                                        \
  41 __extension__ ({                                                        \
  42         rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p));  \
  43         rseq_barrier();                                                 \
  44         ____p1;                                                         \
  45 })
  46
  47 /* Acquire barrier after control dependency. */
  48 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  49
  50 /* Release: One-way permeable barrier. */
  51 #define rseq_smp_store_release(p, v)                                    \
  52 do {                                                                    \
  53         rseq_barrier();                                                 \
  54         RSEQ_WRITE_ONCE(*(p), v);                                       \
  55 } while (0)
  56
  57 #ifdef RSEQ_ARCH_S390X
  58
  59 /*
  60  * Helper macros to access a variable of pointer type stored in a 64-bit
  61  * integer. Only used internally in rseq headers.
  62  */
  63 #define RSEQ_ASM_LONG_L                 "lg"
  64 #define RSEQ_ASM_LONG_S                 "stg"
  65 #define RSEQ_ASM_LONG_LT_R              "ltgr"
  66 #define RSEQ_ASM_LONG_CMP               "cg"
  67 #define RSEQ_ASM_LONG_CMP_R             "cgr"
  68 #define RSEQ_ASM_LONG_ADDI              "aghi"
  69 #define RSEQ_ASM_LONG_ADD_R             "agr"
  70
  71 /* Only used in RSEQ_ASM_DEFINE_TABLE. */
  72 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
  73                                 start_ip, post_commit_offset, abort_ip) \
  74                 ".pushsection __rseq_cs, \"aw\"\n\t"                    \
  75                 ".balign 32\n\t"                                        \
  76                 __rseq_str(label) ":\n\t"                               \
  77                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
  78                 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
  79                 ".popsection\n\t"                                       \
  80                 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
  81                 ".quad " __rseq_str(label) "b\n\t"                      \
  82                 ".popsection\n\t"
  83
  84 /*
  85  * Define the @exit_ip pointer as an exit point for the sequence of consecutive
  86  * assembly instructions at @start_ip.
  87  *
  88  *  @start_ip:
  89  *    Pointer to the first instruction of the sequence of consecutive assembly
  90  *    instructions.
  91  *  @exit_ip:
  92  *    Pointer to an exit point instruction.
  93  *
  94  * Exit points of a rseq critical section consist of all instructions outside
  95  * of the critical section where a critical section can either branch to or
  96  * reach through the normal course of its execution. The abort IP and the
  97  * post-commit IP are already part of the __rseq_cs section and should not be
  98  * explicitly defined as additional exit points. Knowing all exit points is
  99  * useful to assist debuggers stepping over the critical section.
 100  */
 101 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
 102                 ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
 103                 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
 104                 ".popsection\n\t"
 105
 106 #else /* #ifdef RSEQ_ARCH_S390X */
 107
 108 /*
 109  * Helper macros to access a variable of pointer type stored in a 64-bit
 110  * integer. Only used internally in rseq headers.
 111  */
 112 #define RSEQ_ASM_LONG_L                 "l"
 113 #define RSEQ_ASM_LONG_S                 "st"
 114 #define RSEQ_ASM_LONG_LT_R              "ltr"
 115 #define RSEQ_ASM_LONG_CMP               "c"
 116 #define RSEQ_ASM_LONG_CMP_R             "cr"
 117 #define RSEQ_ASM_LONG_ADDI              "ahi"
 118 #define RSEQ_ASM_LONG_ADD_R             "ar"
 119
 120 /* Only used in RSEQ_ASM_DEFINE_TABLE. */
 121 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                  \
 122                                 start_ip, post_commit_offset, abort_ip) \
 123                 ".pushsection __rseq_cs, \"aw\"\n\t"                    \
 124                 ".balign 32\n\t"                                        \
 125                 __rseq_str(label) ":\n\t"                               \
 126                 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
 127                 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
 128                 ".popsection\n\t"                                       \
 129                 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"          \
 130                 ".long 0x0, " __rseq_str(label) "b\n\t"                 \
 131                 ".popsection\n\t"
 132
 133 /*
 134  * Define the @exit_ip pointer as an exit point for the sequence of consecutive
 135  * assembly instructions at @start_ip.
 136  *
 137  *  @start_ip:
 138  *    Pointer to the first instruction of the sequence of consecutive assembly
 139  *    instructions.
 140  *  @exit_ip:
 141  *    Pointer to an exit point instruction.
 142  *
 143  * Exit points of a rseq critical section consist of all instructions outside
 144  * of the critical section where a critical section can either branch to or
 145  * reach through the normal course of its execution. The abort IP and the
 146  * post-commit IP are already part of the __rseq_cs section and should not be
 147  * explicitly defined as additional exit points. Knowing all exit points is
 148  * useful to assist debuggers stepping over the critical section.
 149  */
 150 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
 151                 ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
 152                 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
 153                 ".popsection\n\t"
 154
 155 #endif /* #ifdef RSEQ_ARCH_S390X */
 156
 157 /*
 158  * Define an rseq critical section structure of version 0 with no flags.
 159  *
 160  *  @label:
 161  *    Local label for the beginning of the critical section descriptor
 162  *    structure.
 163  *  @start_ip:
 164  *    Pointer to the first instruction of the sequence of consecutive assembly
 165  *    instructions.
 166  *  @post_commit_ip:
 167  *    Pointer to the instruction after the last instruction of the sequence of
 168  *    consecutive assembly instructions.
 169  *  @abort_ip:
 170  *    Pointer to the instruction where to move the execution flow in case of
 171  *    abort of the sequence of consecutive assembly instructions.
 172  */
 173 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
 174         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
 175                                 (post_commit_ip) - (start_ip), abort_ip)
 176
 177 /*
 178  * Define a critical section abort handler.
 179  *
 180  *  @label:
 181  *    Local label to the abort handler.
 182  *  @teardown:
 183  *    Sequence of instructions to run on abort.
 184  *  @abort_label:
 185  *    C label to jump to at the end of the sequence.
 186  */
 187 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
 188                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 189                 ".long " __rseq_str(RSEQ_SIG) "\n\t"                    \
 190                 __rseq_str(label) ":\n\t"                               \
 191                 teardown                                                \
 192                 "jg %l[" __rseq_str(abort_label) "]\n\t"                \
 193                 ".popsection\n\t"
 194
 195 /*
 196  * Define a critical section teardown handler.
 197  *
 198  *  @label:
 199  *    Local label to the teardown handler.
 200  *  @teardown:
 201  *    Sequence of instructions to run on teardown.
 202  *  @target_label:
 203  *    C label to jump to at the end of the sequence.
 204  */
 205 #define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label)         \
 206                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 207                 __rseq_str(label) ":\n\t"                               \
 208                 teardown                                                \
 209                 "jg %l[" __rseq_str(target_label) "]\n\t"               \
 210                 ".popsection\n\t"
 211
 212 /*
 213  * Store the address of the critical section descriptor structure at
 214  * @cs_label into the @rseq_cs pointer and emit the label @label, which
 215  * is the beginning of the sequence of consecutive assembly instructions.
 216  *
 217  *  @label:
 218  *    Local label to the beginning of the sequence of consecutive assembly
 219  *    instructions.
 220  *  @cs_label:
 221  *    Source local label to the critical section descriptor structure.
 222  *  @rseq_cs:
 223  *    Destination pointer where to store the address of the critical
 224  *    section descriptor structure.
 225  */
 226 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
 227                 RSEQ_INJECT_ASM(1)                                      \
 228                 "larl %%r0, " __rseq_str(cs_label) "\n\t"               \
 229                 RSEQ_ASM_LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t" \
 230                 __rseq_str(label) ":\n\t"
 231
 232 /* Jump to local label @label when @cpu_id != @current_cpu_id. */
 233 #define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label)             \
 234                 RSEQ_INJECT_ASM(2)                                      \
 235                 "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
 236                 "jnz " __rseq_str(label) "\n\t"
 237
 238 /* Per-cpu-id indexing. */
 239
 240 #define RSEQ_TEMPLATE_INDEX_CPU_ID
 241 #define RSEQ_TEMPLATE_MO_RELAXED
 242 #include "rseq/arch/s390/bits.h"
 243 #undef RSEQ_TEMPLATE_MO_RELAXED
 244
 245 #define RSEQ_TEMPLATE_MO_RELEASE
 246 #include "rseq/arch/s390/bits.h"
 247 #undef RSEQ_TEMPLATE_MO_RELEASE
 248 #undef RSEQ_TEMPLATE_INDEX_CPU_ID
 249
 250 /* Per-mm-cid indexing. */
 251
 252 #define RSEQ_TEMPLATE_INDEX_MM_CID
 253 #define RSEQ_TEMPLATE_MO_RELAXED
 254 #include "rseq/arch/s390/bits.h"
 255 #undef RSEQ_TEMPLATE_MO_RELAXED
 256
 257 #define RSEQ_TEMPLATE_MO_RELEASE
 258 #include "rseq/arch/s390/bits.h"
 259 #undef RSEQ_TEMPLATE_MO_RELEASE
 260 #undef RSEQ_TEMPLATE_INDEX_MM_CID
 261
 262 /* APIs which are not indexed. */
 263
 264 #define RSEQ_TEMPLATE_INDEX_NONE
 265 #define RSEQ_TEMPLATE_MO_RELAXED
 266 #include "rseq/arch/s390/bits.h"
 267 #undef RSEQ_TEMPLATE_MO_RELAXED
 268 #undef RSEQ_TEMPLATE_INDEX_NONE