include/rseq/arch/s390.h

   1 /* SPDX-License-Identifier: MIT */
   2 /* SPDX-FileCopyrightText: 2018 Vasily Gorbik <gor@linux.ibm.com> */
   3 /* SPDX-FileCopyrightText: 2024 Mathieu Desnoyers <mathieu.desnoyers@efficios.com> */
   4
   5 /*
   6  * rseq-s390.h
   7  */
   8
   9 /*
  10  * RSEQ_ASM_*() macro helpers are internal to the librseq headers. Those
  11  * are not part of the public API.
  12  */
  13
  14 #ifndef _RSEQ_RSEQ_H
  15 #error "Never use <rseq/arch/s390.h> directly; include <rseq/rseq.h> instead."
  16 #endif
  17
  18 /*
  19  * RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the
  20  * access-register mode nor the linkage stack this instruction will always
  21  * cause a special-operation exception (the trap-enabled bit in the DUCT
  22  * is and will stay 0). The instruction pattern is
  23  *      b2 ff 0f ff     trap4   4095(%r0)
  24  */
  25 #define RSEQ_SIG        0xB2FF0FFF
  26
  27 /*
  28  * Refer to the Linux kernel memory model (LKMM) for documentation of
  29  * the memory barriers.
  30  */
  31
  32 /* CPU memory barrier. */
  33 #define rseq_smp_mb()   __asm__ __volatile__ ("bcr 15,0" ::: "memory")
  34 /* CPU read memory barrier */
  35 #define rseq_smp_rmb()  rseq_smp_mb()
  36 /* CPU write memory barrier */
  37 #define rseq_smp_wmb()  rseq_smp_mb()
  38
  39 /* Acquire: One-way permeable barrier. */
  40 #define rseq_smp_load_acquire(p)                                        \
  41 __extension__ ({                                                        \
  42         rseq_unqual_scalar_typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p));  \
  43         rseq_barrier();                                                 \
  44         ____p1;                                                         \
  45 })
  46
  47 /* Acquire barrier after control dependency. */
  48 #define rseq_smp_acquire__after_ctrl_dep()      rseq_smp_rmb()
  49
  50 /* Release: One-way permeable barrier. */
  51 #define rseq_smp_store_release(p, v)                                    \
  52 do {                                                                    \
  53         rseq_barrier();                                                 \
  54         RSEQ_WRITE_ONCE(*(p), v);                                       \
  55 } while (0)
  56
  57 /*
  58  * Helper macros to access a variable of long integer type. Only used
  59  * internally in rseq headers.
  60  */
  61 #ifdef RSEQ_ARCH_S390X
  62 # define RSEQ_ASM_LONG_L                "lg"
  63 # define RSEQ_ASM_LONG_S                "stg"
  64 # define RSEQ_ASM_LONG_LT_R             "ltgr"
  65 # define RSEQ_ASM_LONG_CMP              "cg"
  66 # define RSEQ_ASM_LONG_CMP_R            "cgr"
  67 # define RSEQ_ASM_LONG_ADDI             "aghi"
  68 # define RSEQ_ASM_LONG_ADD_R            "agr"
  69 #else
  70 # define RSEQ_ASM_LONG_L                "l"
  71 # define RSEQ_ASM_LONG_S                "st"
  72 # define RSEQ_ASM_LONG_LT_R             "ltr"
  73 # define RSEQ_ASM_LONG_CMP              "c"
  74 # define RSEQ_ASM_LONG_CMP_R            "cr"
  75 # define RSEQ_ASM_LONG_ADDI             "ahi"
  76 # define RSEQ_ASM_LONG_ADD_R            "ar"
  77 #endif
  78
  79 #ifdef RSEQ_ARCH_S390X
  80 # define RSEQ_ASM_U64_PTR(x)            ".quad " x
  81 #else
  82 /* 32-bit only supported on big endian. */
  83 # define RSEQ_ASM_U64_PTR(x)            ".long 0x0, " x
  84 #endif
  85
  86 #define RSEQ_ASM_U32(x)                 ".long " x
  87
  88 /* Common architecture support macros. */
  89 #include "rseq/arch/generic/common.h"
  90
  91 /*
  92  * Define a critical section abort handler.
  93  *
  94  *  @label:
  95  *    Local label to the abort handler.
  96  *  @teardown:
  97  *    Sequence of instructions to run on abort.
  98  *  @abort_label:
  99  *    C label to jump to at the end of the sequence.
 100  */
 101 #define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label)             \
 102                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 103                 RSEQ_ASM_U32(__rseq_str(RSEQ_SIG)) "\n\t"               \
 104                 __rseq_str(label) ":\n\t"                               \
 105                 teardown                                                \
 106                 "jg %l[" __rseq_str(abort_label) "]\n\t"                \
 107                 ".popsection\n\t"
 108
 109 /*
 110  * Define a critical section teardown handler.
 111  *
 112  *  @label:
 113  *    Local label to the teardown handler.
 114  *  @teardown:
 115  *    Sequence of instructions to run on teardown.
 116  *  @target_label:
 117  *    C label to jump to at the end of the sequence.
 118  */
 119 #define RSEQ_ASM_DEFINE_TEARDOWN(label, teardown, target_label)         \
 120                 ".pushsection __rseq_failure, \"ax\"\n\t"               \
 121                 __rseq_str(label) ":\n\t"                               \
 122                 teardown                                                \
 123                 "jg %l[" __rseq_str(target_label) "]\n\t"               \
 124                 ".popsection\n\t"
 125
 126 /*
 127  * Store the address of the critical section descriptor structure at
 128  * @cs_label into the @rseq_cs pointer and emit the label @label, which
 129  * is the beginning of the sequence of consecutive assembly instructions.
 130  *
 131  *  @label:
 132  *    Local label to the beginning of the sequence of consecutive assembly
 133  *    instructions.
 134  *  @cs_label:
 135  *    Source local label to the critical section descriptor structure.
 136  *  @rseq_cs:
 137  *    Destination pointer where to store the address of the critical
 138  *    section descriptor structure.
 139  */
 140 #define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs)                \
 141                 RSEQ_INJECT_ASM(1)                                      \
 142                 "larl %%r0, " __rseq_str(cs_label) "\n\t"               \
 143                 RSEQ_ASM_LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t" \
 144                 __rseq_str(label) ":\n\t"
 145
 146 /* Jump to local label @label when @cpu_id != @current_cpu_id. */
 147 #define RSEQ_ASM_CBNE_CPU_ID(cpu_id, current_cpu_id, label)             \
 148                 RSEQ_INJECT_ASM(2)                                      \
 149                 "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
 150                 "jnz " __rseq_str(label) "\n\t"
 151
 152 /* Per-cpu-id indexing. */
 153
 154 #define RSEQ_TEMPLATE_INDEX_CPU_ID
 155 #define RSEQ_TEMPLATE_MO_RELAXED
 156 #include "rseq/arch/s390/bits.h"
 157 #undef RSEQ_TEMPLATE_MO_RELAXED
 158
 159 #define RSEQ_TEMPLATE_MO_RELEASE
 160 #include "rseq/arch/s390/bits.h"
 161 #undef RSEQ_TEMPLATE_MO_RELEASE
 162 #undef RSEQ_TEMPLATE_INDEX_CPU_ID
 163
 164 /* Per-mm-cid indexing. */
 165
 166 #define RSEQ_TEMPLATE_INDEX_MM_CID
 167 #define RSEQ_TEMPLATE_MO_RELAXED
 168 #include "rseq/arch/s390/bits.h"
 169 #undef RSEQ_TEMPLATE_MO_RELAXED
 170
 171 #define RSEQ_TEMPLATE_MO_RELEASE
 172 #include "rseq/arch/s390/bits.h"
 173 #undef RSEQ_TEMPLATE_MO_RELEASE
 174 #undef RSEQ_TEMPLATE_INDEX_MM_CID
 175
 176 /* APIs which are not indexed. */
 177
 178 #define RSEQ_TEMPLATE_INDEX_NONE
 179 #define RSEQ_TEMPLATE_MO_RELAXED
 180 #include "rseq/arch/s390/bits.h"
 181 #undef RSEQ_TEMPLATE_MO_RELAXED
 182 #undef RSEQ_TEMPLATE_INDEX_NONE