include/rseq/arch/s390.h

   1 /* SPDX-License-Identifier: MIT */
   2 /* SPDX-FileCopyrightText: 2018 Vasily Gorbik <gor@linux.ibm.com> */
   3 /* SPDX-FileCopyrightText: 2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
   4
   5 /*
   6  * rseq-s390.h
   7  */
   8
   9 /*
  10  * RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those
  11  * are not part of the public API.
  12  */
  13
  14 #ifndef _RSEQ_RSEQ_H
  15 #error "Never use <rseq/arch/s390.h> directly; include <rseq/rseq.h> instead."
  16 #endif
  17
  18 /*
  19  * RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the
  20  * access-register mode nor the linkage stack this instruction will always
  21  * cause a special-operation exception (the trap-enabled bit in the DUCT
  22  * is and will stay 0). The instruction pattern is
  23  *      b2 ff 0f ff     trap4   4095(%r0)
  24  */
  25 #define RSEQ_SIG        0xB2FF0FFF
  26
  27 /*
  28  * Refer to the Linux kernel memory model (LKMM) for documentation of
  29  * the memory barriers.
  30  */
  31
  32 /* CPU memory barrier. */
  33 #define rseq_smp_mb()   __asm__ __volatile__ ("bcr 15,0" ::: "memory")
  34 /* CPU read memory barrier */
  35 #define rseq_smp_rmb()  rseq_smp_mb()
  36 /* CPU write memory barrier */
  37 #define rseq_smp_wmb()  rseq_smp_mb()
  38
  39 /* Acquire: One-way permeable barrier. */
  40 #define rseq_smp_load_acquire(p)                                        \
  41 __extension__ ({                                                        \
  42         rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p));  \
  43         rseq_barrier();                                                 \
  44         ____p1;                                                         \
  45 })
  46
  47 /* Acquire barrier after control dependency. */
  48 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  49
  50 /* Release: One-way permeable barrier. */
  51 #define rseq_smp_store_release(p, v)                                    \
  52 do {                                                                    \
  53         rseq_barrier();                                                 \
  54         RSEQ_WRITE_ONCE(*(p), v);                                       \
  55 } while (0)
  56
  57 /*
  58  * Helper macros to access a variable of long integer type. Only used
  59  * internally in rseq headers.
  60  */
  61 #ifdef RSEQ_ARCH_S390X
  62 # define RSEQ_ASM_LONG_L                "lg"
  63 # define RSEQ_ASM_LONG_S                "stg"
  64 # define RSEQ_ASM_LONG_LT_R             "ltgr"
  65 # define RSEQ_ASM_LONG_CMP              "cg"
  66 # define RSEQ_ASM_LONG_CMP_R            "cgr"
  67 # define RSEQ_ASM_LONG_ADDI             "aghi"
  68 # define RSEQ_ASM_LONG_ADD_R            "agr"
  69 #else
  70 # define RSEQ_ASM_LONG_L                "l"
  71 # define RSEQ_ASM_LONG_S                "st"
  72 # define RSEQ_ASM_LONG_LT_R             "ltr"
  73 # define RSEQ_ASM_LONG_CMP              "c"
  74 # define RSEQ_ASM_LONG_CMP_R            "cr"
  75 # define RSEQ_ASM_LONG_ADDI             "ahi"
  76 # define RSEQ_ASM_LONG_ADD_R            "ar"
  77 #endif
  78
  79 #ifdef RSEQ_ARCH_S390X
  80 # define RSEQ_ASM_U64_PTR(x)            ".quad " x
  81 #else
  82 /* 32-bit only supported on big endian. */
  83 # define RSEQ_ASM_U64_PTR(x)            ".long 0x0, " x
  84 #endif
  85
  86 #define RSEQ_ASM_U32(x)                 ".long " x
  87
  88 /* Only used in RSEQ_ASM_DEFINE_TABLE. */
  89 #define __RSEQ_ASM_DEFINE_TABLE(label, version, flags,                          \
  90                         start_ip, post_commit_offset, abort_ip)                 \
  91                 ".pushsection __rseq_cs, \"aw\"\n\t"                            \
  92                 ".balign 32\n\t"                                                \
  93                 __rseq_str(label) ":\n\t"                                       \
  94                 RSEQ_ASM_U32(__rseq_str(version)) "\n\t"                \
  95                 RSEQ_ASM_U32(__rseq_str(flags)) "\n\t"                  \
  96                 RSEQ_ASM_U64_PTR(__rseq_str(start_ip)) "\n\t"                   \
  97                 RSEQ_ASM_U64_PTR(__rseq_str(post_commit_offset)) "\n\t"         \
  98                 RSEQ_ASM_U64_PTR(__rseq_str(abort_ip)) "\n\t"                   \
  99                 ".popsection\n\t"                                               \
 100                 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t"                  \
 101                 RSEQ_ASM_U64_PTR(__rseq_str(label) "b") "\n\t"                  \
 102                 ".popsection\n\t"
 103
 104 /*
 105  * Define an rseq critical section structure of version 0 with no flags.
 106  *
 107  *  @label:
 108  *    Local label for the beginning of the critical section descriptor
 109  *    structure.
 110  *  @start_ip:
 111  *    Pointer to the first instruction of the sequence of consecutive assembly
 112  *    instructions.
 113  *  @post_commit_ip:
 114  *    Pointer to the instruction after the last instruction of the sequence of
 115  *    consecutive assembly instructions.
 116  *  @abort_ip:
 117  *    Pointer to the instruction where to move the execution flow in case of
 118  *    abort of the sequence of consecutive assembly instructions.
 119  */
 120 #define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
 121         __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip,              \
 122                                 (post_commit_ip) - (start_ip), abort_ip)
 123
 124 /*
 125  * Define the @exit_ip pointer as an exit point for the sequence of consecutive
 126  * assembly instructions at @start_ip.
 127  *
 128  *  @start_ip:
 129  *    Pointer to the first instruction of the sequence of consecutive assembly
 130  *    instructions.
 131  *  @exit_ip:
 132  *    Pointer to an exit point instruction.
 133  *
 134  * Exit points of a rseq critical section consist of all instructions outside
 135  * of the critical section where a critical section can either branch to or
 136  * reach through the normal course of its execution. The abort IP and the
 137  * post-commit IP are already part of the __rseq_cs section and should not be
 138  * explicitly defined as additional exit points. Knowing all exit points is
 139  * useful to assist debuggers stepping over the critical section.
 140  */
 141 #define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip)                   \
 142                 ".pushsection __rseq_exit_point_array, \"aw\"\n\t"      \
 143                 RSEQ_ASM_U64_PTR(__rseq_str(start_ip)) "\n\t"           \
 144                 RSEQ_ASM_U64_PTR(__rseq_str(exit_ip)) "\n\t"            \
 145                 ".popsection\n\t"
 146
 147 /*
 148  * Define a critical section abort handler.
 149  *
 150  *  @label:
 151  *    Local label to the abort handler.
 152  *  @teardown:
 153  *    Sequence of instructions to run on abort.
 154  *  @abort_label:
 155  *    C label to jump to at the end of the sequence.
 156  */
 157 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
 158                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 159                 RSEQ_ASM_U32(__rseq_str(RSEQ_SIG)) "\n\t"               \
 160                 __rseq_str(label) ":\n\t"                               \
 161                 teardown                                                \
 162                 "jg %l[" __rseq_str(abort_label) "]\n\t"                \
 163                 ".popsection\n\t"
 164
 165 /*
 166  * Define a critical section teardown handler.
 167  *
 168  *  @label:
 169  *    Local label to the teardown handler.
 170  *  @teardown:
 171  *    Sequence of instructions to run on teardown.
 172  *  @target_label:
 173  *    C label to jump to at the end of the sequence.
 174  */
 175 #define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label)         \
 176                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 177                 __rseq_str(label) ":\n\t"                               \
 178                 teardown                                                \
 179                 "jg %l[" __rseq_str(target_label) "]\n\t"               \
 180                 ".popsection\n\t"
 181
 182 /*
 183  * Store the address of the critical section descriptor structure at
 184  * @cs_label into the @rseq_cs pointer and emit the label @label, which
 185  * is the beginning of the sequence of consecutive assembly instructions.
 186  *
 187  *  @label:
 188  *    Local label to the beginning of the sequence of consecutive assembly
 189  *    instructions.
 190  *  @cs_label:
 191  *    Source local label to the critical section descriptor structure.
 192  *  @rseq_cs:
 193  *    Destination pointer where to store the address of the critical
 194  *    section descriptor structure.
 195  */
 196 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
 197                 RSEQ_INJECT_ASM(1)                                      \
 198                 "larl %%r0, " __rseq_str(cs_label) "\n\t"               \
 199                 RSEQ_ASM_LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t" \
 200                 __rseq_str(label) ":\n\t"
 201
 202 /* Jump to local label @label when @cpu_id != @current_cpu_id. */
 203 #define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label)             \
 204                 RSEQ_INJECT_ASM(2)                                      \
 205                 "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
 206                 "jnz " __rseq_str(label) "\n\t"
 207
 208 /* Per-cpu-id indexing. */
 209
 210 #define RSEQ_TEMPLATE_INDEX_CPU_ID
 211 #define RSEQ_TEMPLATE_MO_RELAXED
 212 #include "rseq/arch/s390/bits.h"
 213 #undef RSEQ_TEMPLATE_MO_RELAXED
 214
 215 #define RSEQ_TEMPLATE_MO_RELEASE
 216 #include "rseq/arch/s390/bits.h"
 217 #undef RSEQ_TEMPLATE_MO_RELEASE
 218 #undef RSEQ_TEMPLATE_INDEX_CPU_ID
 219
 220 /* Per-mm-cid indexing. */
 221
 222 #define RSEQ_TEMPLATE_INDEX_MM_CID
 223 #define RSEQ_TEMPLATE_MO_RELAXED
 224 #include "rseq/arch/s390/bits.h"
 225 #undef RSEQ_TEMPLATE_MO_RELAXED
 226
 227 #define RSEQ_TEMPLATE_MO_RELEASE
 228 #include "rseq/arch/s390/bits.h"
 229 #undef RSEQ_TEMPLATE_MO_RELEASE
 230 #undef RSEQ_TEMPLATE_INDEX_MM_CID
 231
 232 /* APIs which are not indexed. */
 233
 234 #define RSEQ_TEMPLATE_INDEX_NONE
 235 #define RSEQ_TEMPLATE_MO_RELAXED
 236 #include "rseq/arch/s390/bits.h"
 237 #undef RSEQ_TEMPLATE_MO_RELAXED
 238 #undef RSEQ_TEMPLATE_INDEX_NONE